diff options
| author | Doug Ledford <dledford@redhat.com> | 2017-01-17 14:42:53 -0500 |
|---|---|---|
| committer | Doug Ledford <dledford@redhat.com> | 2017-01-17 14:42:53 -0500 |
| commit | 1a6ab7f4c4aa048e8cf0c6cbed5935181f660bd8 (patch) | |
| tree | 546f94ecb8adb3120c27e8ac2fbeb8b068f45c01 | |
| parent | 3b3973377f0dbccb11323b86e267308052cc4458 (diff) | |
Now that all of the rdma components have been pulled into a single
repository on github, remove all of the contents of this repo and
leave a pointer to the new location.
Signed-off-by: Doug Ledford <dledford@redhat.com>
112 files changed, 10 insertions, 23110 deletions
diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index fcea350..0000000 --- a/AUTHORS +++ /dev/null @@ -1,4 +0,0 @@ -Roland Dreier <roland@topspin.com> -Dotan Barak <dotanba@gmail.com> -Sean Hefty <sean.hefty@intel.com> -Michael S. Tsirkin <mst@mellanox.co.il> diff --git a/COPYING b/COPYING deleted file mode 100644 index ee1a79f..0000000 --- a/COPYING +++ /dev/null @@ -1,378 +0,0 @@ -This software is available to you under a choice of one of two -licenses. You may choose to be licensed under the terms of the the -OpenIB.org BSD license or the GNU General Public License (GPL) Version -2, both included below. - -Copyright (c) 2004 Topspin Communications. All rights reserved. - -================================================================== - - OpenIB.org BSD license - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -================================================================== - - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index 27ee956..0000000 --- a/ChangeLog +++ /dev/null @@ -1,583 +0,0 @@ -2006-10-30 Jack Morgenstein <jackm@mellanox.co.il> - - * src/cmd.c (ibv_cmd_query_qp): Unmarshall sq_draining instead of - en_sqd_async_notify. - - * include/infiniband/kern-abi.h: Change en_sqd_async_notify member - of struct ibv_query_qp_resp to sq_draining. - -2006-10-30 Roland Dreier <rdreier@cisco.com> - - * src/init.c (find_drivers): Make find_drivers() take a const - directory name, and tweak how we strip trailing /s so that we - don't have to modify the directory name passed in. Constify - default_path too. - -2006-10-25 Roland Dreier <rdreier@cisco.com> - - * src/init.c (init_drivers): Remove assignment to dev->driver now - that it is gone for good. - - * include/infiniband/verbs.h: Remove .driver member of struct - ibv_device, since it is never really used. - -2006-10-17 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/arch.h: Update i386 and x86_64 memory barrier - macros to be more than compiler barriers, to guard against - out-of-order speculative reads. - - * include/infiniband/arch.h: Add rmb() and wmb() macros in - addition to the full mb(), so that low-level drivers can ask for - weaker ordering if that's all that is needed. - -2006-10-03 Roland Dreier <rdreier@cisco.com> - - * src/cmd.c (ibv_cmd_get_context_v2, ibv_cmd_get_context) - (ibv_cmd_query_device, ibv_cmd_query_port, ibv_cmd_alloc_pd) - (ibv_cmd_reg_mr, ibv_cmd_create_cq_v2, ibv_cmd_create_cq) - (ibv_cmd_poll_cq, ibv_cmd_resize_cq, ibv_cmd_destroy_cq) - (ibv_cmd_create_srq, ibv_cmd_create_qp, ibv_cmd_post_send) - (ibv_cmd_post_recv, ibv_cmd_post_srq_recv, ibv_cmd_create_ah) - (ibv_cmd_destroy_qp): Annotate so that Valgrind knows responses - are defined after write() succeeds. The kernel writes into the - response structure directly, so without these, Valgrind thinks - that response structures are undefined memory. This is based on - patches and suggestions by Rainer Keller <keller@hlrs.de>, Jeff - Squyres <jsquyres@cisco.com> and Siqing Fan. - - * src/ibverbs.h: Add wrapper for VALGRIND_MAKE_MEM_DEFINED so that - it can be used in .c files without worrying about whether Valgrind - is installed or enabled. - - * configure.in: Add support for Valgrind annotation (enabled with - --with-valgrind option to configure). - - * src/cmd.c (ibv_cmd_query_port, ibv_cmd_create_cq, - ibv_cmd_modify_qp): Set reserved fields to 0 to avoid future - problems and also to make Valgrind a little quieter. - - * src/init.c (init_drivers): Set node_type and transport_type - values of device being created. - - * include/infiniband/verbs.h: Add ibv_node_type enum value - IBV_NODE_RNIC, and add enum ibv_transport_type. Add node_type and - transport_type fields to struct ibv_device. - -2006-09-12 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h: Swap wr_id and next members of - struct ibv_send_wr and struct ibv_recv_wr. This allows wr_id to - be naturally aligned without padding on 32-bit platforms. - -2006-08-23 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/driver.h: Add a definition of the macro - IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS so that low-level driver plugins - can detect the changed signature of ibv_cmd_resize_cq(). - -2006-08-23 Ralph Campbell <ralph.campbell@qlogic.com> - - * src/cmd.c (ibv_cmd_resize_cq): Add resp and resp_size parameters - so that the low-level driver in the kernel can return - device-specific information from the resize CQ operation. - -2006-07-26 Roland Dreier <rdreier@cisco.com> - - * src/verbs.c (ibv_reg_mr, ibv_dereg_mr): Add calls to - ibv_dontfork_range() and ibv_dofork_range() for memory regions - registered by library consumers. - - * include/infiniband/verbs.h: Add declaration of ibv_fork_init(). - - * include/infiniband/driver.h: Add declarations of - ibv_dontfork_range() and ibv_dofork_range(). - - * src/memory.c: Rewrite to use a red-black tree instead of a - linked list. Change from doing mlock()/munlock() to - madvise(..., MADV_DONTFORK) and madvise(..., MADV_DOFORK), and - change the name of the entry points to ibv_dontfork_range() and - ibv_dofork_range(). Add ibv_fork_init() for applications to - request fork-safe behavior. - - * src/ibverbs.h: Kill off unused declarations. - - * src/init.c (ibverbs_init): Get rid of call to ibv_init_mem_map(). - - * include/infiniband/verbs.h: Add addr and length field to struct - ibv_mr so that memory regions can be madvised(). This changes the - ABI, since the layout of struct ibv_mr is changed. - -2006-07-04 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/arch.h: Fix typo in sparc mb() - implementation: the asm should just be empty -- the "sync" - instruction was mistakenly cut and pasted from the ppc version. - -2006-06-07 Sean Hefty <sean.hefty@intel.com> - - * src/verbs.c include/infiniband/verbs.h: Add new routines: - ibv_init_ah_from_wc() and ibv_create_ah_from_wc() to simplify UD QP - communication. - - * src/marshall.c include/infiniband/marshall.h: Expose - ibv_copy_ah_attr_from_kern to retrieve ibv_ah_attr from kernel for - a UD QP. - -2006-06-01 Roland Dreier <rdreier@cisco.com> - - * src/device.c (ibv_get_device_list): Actually return a - NULL-terminated array as the documentation promises. - -2006-05-31 Roland Dreier <rdreier@cisco.com> - - * src/init.c (find_drivers): Fix memory leak: the result of - asprintf() needs to be freed when we're done with it. - - * examples/asyncwatch.c (event_name_str): Print human-readable - form of IBV_EVENT_CLIENT_REREGISTER. - -2006-05-31 Leonid Arsh <leonida@voltaire.com> - - * include/infiniband/verbs.h: Add IBV_EVENT_CLIENT_REREGISTER. - -2006-05-22 Roland Dreier <rdreier@cisco.com> - - * examples/devinfo.c (print_hca_cap): Read board_id attribute from - sysfs using ibv_read_sysfs_file() instead of libsysfs. - - * src/cmd.c, src/marshall.c, src/sysfs.c: Include <string.h>, - since it is no longer implicitly included via <sysfs/libsysfs.h>. - - * include/infiniband/driver.h, include/infiniband/verbs.h, - src/device.c, src/init.c, src/verbs.c: Remove dependency on - libsysfs by implementing what is required directly on top of - filesystem operations. - - * include/infiniband/driver.h, src/init.c: Change name of driver - entry point to ibv_driver_init(), and update prototype to remove - libsysfs dependency. - - * src/marshall.c, include/infiniband/marshall.h, - include/infiniband/sa.h: Remove deprecated ib_xxx symbols. - - * Makefile.am: Bump SONAME to 2, since libibverbs 1.1 will be - ABI-incompatible with libibverbs 1.0. - - * Create libibverbs 1.1 branch and bump version number to 1.1-pre1. - -2006-05-22 Michael S. Tsirkin <mst@mellanox.co.il> - - * include/infiniband/verbs.h: Remove trailing commas from - enumerators to quiet warnings from obsolete compilers. - -2006-05-02 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0.3. - -2006-05-01 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/arch.h: Only SPARC V9 ISA supports membar. - So just use generic memory barrier for older sparc archs. - -2006-04-11 Roland Dreier <rdreier@cisco.com> - - * src/sysfs.c (ibv_read_sysfs_file): Fix memory leak if open fails. - - * src/device.c (ibv_get_device_guid), src/verbs.c (ibv_query_gid, - ibv_query_pkey), src/init.c (init_drivers, check_abi_version): Use - libibverbs functions instead of libsysfs functions to get to sysfs. - - * src/sysfs.c (ibv_get_sysfs_path, ibv_read_sysfs_file): Add some - simple functions for accessing sysfs without using libsysfs. - - * include/infiniband/sa-kern-abi.h: Deprecate struct - ib_kern_path_rec name; struct ibv_kern_path_rec is now preferred. - - * include/infiniband/sa.h: Deprecate struct ib_sa_XXX names; - struct ibv_sa_XXX is now preferred. - - * src/marshall.c, include/infiniband/marshall.h: Deprecate - ib_copy_XXX() names; ibv_copy_XXX() is preferred. Add stub - wrappers with the old names so old binaries still work. - -2006-04-11 Hoang-Nam Nguyen <HNGUYEN@de.ibm.com> - - * src/verbs.c (ibv_rate_to_mult, mult_to_ibv_rate): Add new - functions to convert between IB rate enums and multiples of the - base 2.5 Gb/sec rate. - -2006-04-11 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h: Add __attribute_const macro to - portably mark functions as __attribute__((const)) - -2006-03-28 Roland Dreier <rdreier@cisco.com> - - * src/init.c (load_driver): Print warning if dlopen() of a driver - plugin fails. - -2006-03-22 Dotan Barak <dotanb@mellanox.co.il> - - * examples/asyncwatch.c: Print asynchronous event name as well as - raw integer value. - -2006-03-22 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h (ibv_req_notify_cq): Document - parameters better. - -2006-03-16 Roland Dreier <rdreier@cisco.com> - - * src/cmd.c, src/device.c, src/memory.c, src/verbs.c: Add include - of <stdlib.h> to get a declaration of free() and avoid compile - warnings. - -2006-03-14 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0.2. - - * Makefile.am (EXTRA_DIST): Remove debian/ directory from - tarballs, since Debian policy is that upstream tarballs should not - include it. - -2006-03-13 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0.1. - - * src/init.c (check_abi_version), src/verbs.c (ibv_query_gid, - ibv_query_pkey): Use sysfs_open_attribute() and - sysfs_read_attribute() instead of the deprecated function - sysfs_read_attribute_value(), which is no longer present in - libsysfs2 (which is already in Debian and Ubuntu). - - * Release version 1.0. - -2006-03-06 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h: Add enum ibv_rate to define encoding - of static_rate field (based on a patch from Jack Morgenstein - <jackm@mellanox.co.il>). - -2006-03-06 Ralph Campbell <ralphc@pathscale.com> - - * src/init.c (find_drivers): Fix minor memory leak: call - globfree() to free memory allocated by glob(). - -2006-02-23 Dotan Barak <dotanb@mellanox.co.il> - - * src/cmd.c (ibv_cmd_create_srq): Add support for kernel ABI - version 6 (take SRQ capacity from kernel response to create SRQ). - -2006-02-16 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0-rc7. - - * src/cmd.c (ibv_cmd_create_qp): Add support for kernel ABI - version 5 (properly aligned struct ibv_create_qp_resp). - -2006-02-15 Roland Dreier <rdreier@cisco.com> - - * src/cmd.c (ibv_cmd_create_qp): Allow userspace device-specific - driver to pass in a response buffer, so that the low-level driver - in the kernel can pass back device-specific information. This - changes the userspace driver API, since the signature of - ibv_cmd_create_qp() is changed. - -2006-02-14 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0-rc6. - -2006-02-13 Dotan Barak <dotanb@mellanox.co.il> - - * examples/devinfo.c (print_hca_cap): Print board_id from sysfs, - if present. - -2006-02-13 Roland Dreier <rdreier@cisco.com> - - * examples/asyncwatch.c, examples/device_list.c, - examples/devinfo.c: Remove cpu_to_be64()/be64_to_cpu() and use - htonll()/ntohll() from <infiniband/arch.h>. - -2006-02-13 Dotan Barak <dotanb@mellanox.co.il> - - * src/cmd.c (ibv_cmd_query_qp, ibv_cmd_query_srq), - include/infiniband/driver.h: Add driver interface for calling - query QP and query SRQ kernel commands. - - * include/infiniband/kern-abi.h: Add kernel ABI for query QP and - query SRQ. - - * src/verbs.c (ibv_query_qp, ibv_query_srq), - include/infiniband/verbs.h: Add query QP and query SRQ library - APIs. This changes the provider ABI, since new fields are added - to struct ibv_context_ops; source compatibility with provider - libraries is preserved, but binaries will have to be recompiled. - Neither source nor binary compatibility with consumers of - libibverbs is affected. - -2006-02-01 Roland Dreier <rdreier@cisco.com> - - * examples/rc_pingpong.c, examples/uc_pingpong.c, - examples/ud_pingpong.c, examples/srq_pingpong.c: Fix bug in - searching for device by name when there's more than one device. - -2006-01-31 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h, include/infiniband/driver.h: Remove - useless "extern" from function declarations. - -2006-01-26 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/driver.h, src/cmd.c (ibv_cmd_resize_cq): Add - driver interface for calling resize CQ kernel command. - - * include/infiniband/kern-abi.h: Add resize CQ kernel ABI. - - * include/infiniband/verbs.h, src/verbs.c (ibv_resize_cq): Add - resize CQ library API. This changes the provider ABI, since a new - field is added to struct ibv_context_ops; source compatibility - with provider libraries is preserved, but binaries will have to be - recompiled. Neither source nor binary compatibility with - consumers of libibverbs is affected. - -2006-01-25 Roland Dreier <rdreier@cisco.com> - - * examples/pingpong.c, examples/pingpong.h, - examples/rc_pingpong.c, examples/uc_pingpong.c, - examples/srq_pingpong.c: Move pp_get_local_lid() to pingpong.c to - reduce code duplication. - -2006-01-22 Roland Dreier <rdreier@cisco.com> - - * Release version 1.0-rc5. - -2006-01-22 Dotan Barak <dotanb@mellanox.co.il> - - * examples/devinfo.c (main): Make ibv_devinfo list all IB devices - by default, rather than the first device only. - -2006-01-20 Roland Dreier <rdreier@cisco.com> - - * examples/rc_pingpong.c, examples/uc_pingpong.c, - examples/srq_pingpong.c: Add "-m/--mtu=" option to set path MTU. - (Based on a patch from Ralph Campbell <ralphc@pathscale.com>) - - * examples/pingpong.c, examples/pingpong.h: Create generic - pingpong files so that we can start factoring out common code from - the pingpong examples. Start with functions to convert MTU to an - IBV enum value. - -2006-01-17 Ralph Campbell <ralphc@pathscale.com> - - * examples/rc_pingpong.c (main), examples/srq_pingpong.c (main), - examples/uc_pingpong.c (main), examples/ud_pingpong.c (main): Fix - race when using CQ events by arming CQ before allowing remote side - to start sending. - -2006-01-06 Roland Dreier <rdreier@cisco.com> - - * examples/srq_pingpong.c (main): Fix SRQ example to avoid - problems with many QPs and events. Based on a patch from Dotan - Barak (who also found the problem). - -2006-01-06 Ralph Campbell <ralphc@pathscale.com> - - * examples/rc_pingpong.c (main), examples/srq_pingpong.c (main), - examples/uc_pingpong.c (main), examples/ud_pingpong.c (main): Fix - test of return value of ibv_poll_cq(). - -2006-01-04 Dotan Barak <dotanb@mellanox.co.il> - - * include/infiniband/verbs.h: Fix mask names in description of - ibv_modify_srq. - -2006-01-04 Michael S. Tsirkin <mst@mellanox.co.il> - - * src/init.c (ibverbs_init): Fix ibverbs_init for multiple adapters. - Noted by Christoph Raisch. - -2005-12-15 Roland Dreier <rdreier@cisco.com> - - * include/infiniband/verbs.h: Document that devices must be opened - before calling ibv_free_device_list(). - - * src/verbs.c (ibv_create_srq): Not all provider libraries will - support SRQs, so check if the create_srq method is defined before - calling it. (Based on a patch from Shirley Ma <xma@us.ibm.com>) - -2005-11-11 Roland Dreier <roland@cisco.com> - - * examples/asyncwatch.c, examples/rc_pingpong.c, - examples/srq_pingpong.c, examples/uc_pingpong.c, - examples/ud_pingpong.c, examples/device_list.c, - examples/devinfo.c: Update examples to match new API. - - * include/infiniband/verbs.h, src/device.c, src/init.c, - src/ibverbs.h: Change from dlist-based ibv_get_devices() API to - simpler ibv_get_device_list() and ibv_free_device_list() API. - -2005-11-10 Sean Hefty <sean.hefty@intel.com> - - * include/infiniband/sa-kern-abi.h: New include file to contain - definitions of SA structures passed between userspace and kernel. - - * include/infiniband/sa.h: New include file for definitions of - SA structures used by multiple libraries. - - * include/infiniband/marshall.h src/marshall.c: New files to define - routines used to exchange data with kernel modules. - - * include/infiniband/kern-abi.h: Added data structures used to exchange - QP attribute with kernel modules. - -2005-11-09 Michael S. Tsirkin <mst@mellanox.co.il> - - * src/device.c (ibv_get_devices): Make function reentrant by using - a mutex to make sure we initialize the device list at most once. - -2005-11-08 Roland Dreier <roland@cisco.com> - - * src/cmd.c (ibv_cmd_create_qp): Add handling for new create QP - interface, which has the kernel return QP capabilities. - - * src/cmd.c (ibv_cmd_modify_srq): Split off handling of modify SRQ - for ABI versions 3 and older, which passed max_sge as part of command. - -2005-10-30 Roland Dreier <roland@cisco.com> - - * examples/srq_pingpong.c (pp_init_ctx): Create CQ with rx_depth + - num_qp entries, instead of just rx_depth + 1 entries, because - there can be one send completion pending for each QP. - -2005-10-25 Roland Dreier <roland@cisco.com> - - * Release version 1.0-rc4. - - * examples/uc_pingpong.c (pp_connect_ctx): Fix QP attribute masks - used to modify QP to RTR and RTS -- we should not be setting - RDMA/atomic attributes for UC QPs. Now that the mthca kernel - driver bug is fixed, the error is exposed here. - - * examples/rc_pingpong.c, examples/srq_pingpong.c, - examples/uc_pingpong.c, examples/ud_pingpong.c: Keep track of - whether send and/or receive is pending. This avoids failures when - the remote side receives data and posts a send very quickly, and - the local side completes the receive before the previous send. - With the old code, this could result in posting a send before the - previous send completed, and therefore overrun the send queue. - -2005-10-23 Roland Dreier <roland@cisco.com> - - * src/cmd.c (ibv_cmd_get_context_v2): Correct silly mistake in - computation of size of buffer for old ABI command: we need to use - sizeof *cmd instead of sizeof cmd, since cmd is a pointer. - -2005-10-21 Roland Dreier <roland@cisco.com> - - * src/cmd.c (ibv_cmd_post_send, ibv_cmd_post_recv, - ibv_cmd_post_srq_recv): Correct value that we check write() return - value against so that we check against the size we actually try to - write, instead of just sizeof cmd. - -2005-10-19 Roland Dreier <roland@cisco.com> - - * src/cmd.c (ibv_cmd_req_notify_cq): Correct how we pass - solicited_only flag into the kernel. - -2005-10-13 Roland Dreier <roland@cisco.com> - - * include/infiniband/driver.h, src/cmd.c, src/libibverbs.map: Add - command functions for calling new kernel commands. - - * include/infiniband/verbs.h: Add qp_type to struct ibv_qp so that - we know when we're posting a send on a UD QP, and add kernel - handle member to struct ibv_ah so we can handle drivers that do - create AH and destroy AH operations in the kernel. - - * include/infiniband/kern-abi.h: Add new command structures for - poll CQ, request notification for CQ, post send, post receive, - post SRQ receive, create AH and destroy AH commands. These will - be used by the PathScale userspace driver. - -2005-10-12 Roland Dreier <roland@cisco.com> - - * examples/srq_pingpong.c (main): Zero out unused entries in - my_dest array to avoid string overflows when we send to the other - side. - -2005-10-09 Roland Dreier <roland@cisco.com> - - * examples/devinfo.c (print_hca_cap): Only print max_mr_size and - page_size_cap if verbose is set. - -2005-10-05 Roland Dreier <roland@cisco.com> - - * src/cmd.c (ibv_cmd_modify_srq): Add function for marshalling - modify SRQ command. - -2005-09-29 Roland Dreier <roland@cisco.com> - - * examples/devinfo.c (print_hca_cap): Get rid of formatting of - firmware version in what should be device-independent code. - - * include/infiniband/driver.h, include/infiniband/verbs.h, - src/cmd.c (ibv_cmd_query_device): Change firmware version in - struct ibv_device_attr to be a string formatted by device-specific - library. - -2005-09-25 Roland Dreier <roland@cisco.com> - - * examples/rc_pingpong.c, examples/srq_pingpong.c, - examples/uc_pingpong.c, examples/ud_pingpong.c: Update to match - new completion channel and CQ creation API. - - * include/infiniband/driver.h, include/infiniband/verbs.h, - src/device.c, src/ibverbs.h, src/verbs.c, src/cmd.c: Add notion of - "completion channel" that allows consumers to dynamically create - and destroy file descriptors for retrieving completion events. - Completion channels are handled natively with kernel ABI version 3 - and simulated with backwards compatibility implementations for ABI - versions 1 and 2. - - * include/infiniband/kern-abi.h: Update to match kernel ABI - version 3. - -2005-09-07 Roland Dreier <roland@cisco.com> - - * src/device.c (ibv_get_device_guid): Use htonll() instead of - relying on pointer aliasing (which seems to break for some gcc - versions). - - * include/infiniband/arch.h: Add htonll() and ntohll() functions. - -2005-09-06 Roland Dreier <roland@cisco.com> - - * include/infiniband/kern-abi.h, include/infiniband/verbs.h, - src/cmd.c, src/device.c, src/verbs.c, examples/asyncwatch.c: - Update to handle new kernel ABI for avoiding stale completion - events. This is completely analogous to the previous asynchronous - event change. - -2005-08-31 Roland Dreier <roland@cisco.com> - - * include/infiniband/kern-abi.h, include/infiniband/verbs.h, - src/cmd.c, src/device.c, src/ibverbs.h, src/init.c, src/verbs.c, - examples/asyncwatch.c: Update to handle new kernel ABI for - avoiding stale asynchronous events. When a CQ, QP or SRQ is - destroyed, the kernel reports the number of events it has given to - userspace, and we wait until we've handled the same number of - events. - - This does introduce a library API change: consumers are now - required to call ibv_put_async_event() to release every - asynchronous event that they retrieve via ibv_get_async_event(). - -2005-08-30 Roland Dreier <roland@cisco.com> - - * man/ibv_asyncwatch.1, man/ibv_devices.1, man/ibv_devinfo.1, - man/ibv_rc_pingpong.1, man/ibv_srq_pingpong.1, - man/ibv_uc_pingpong.1, man/ibv_ud_pingpong.1: Add man pages for - example programs. - - * examples/devinfo.c: Merge with Dotan Barak's vstat tool. diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index 1361e96..0000000 --- a/Makefile.am +++ /dev/null @@ -1,132 +0,0 @@ -AM_CPPFLAGS = -I$(srcdir)/include - -lib_LTLIBRARIES = src/libibverbs.la - -ACLOCAL_AMFLAGS = -I config -AM_CFLAGS = -g -Wall -fno-strict-aliasing - -src_libibverbs_la_CFLAGS = $(AM_CFLAGS) -DIBV_CONFIG_DIR=\"$(sysconfdir)/libibverbs.d\" \ - $(LIBNL_CFLAGS) -libibverbs_version_script = @LIBIBVERBS_VERSION_SCRIPT@ -src_libibverbs_la_LIBADD = $(LIBNL_LIBS) - -src_libibverbs_la_SOURCES = src/cmd.c src/compat-1_0.c src/device.c src/init.c \ - src/marshall.c src/memory.c src/sysfs.c src/verbs.c \ - src/enum_strs.c -if ! NO_RESOLVE_NEIGH -src_libibverbs_la_SOURCES += src/neigh.c -noinst_HEADERS = src/neigh.h src/nl1_compat.h -endif -src_libibverbs_la_LDFLAGS = -version-info 1 -export-dynamic \ - $(libibverbs_version_script) -src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map - -bin_PROGRAMS = examples/ibv_devices examples/ibv_devinfo \ - examples/ibv_asyncwatch examples/ibv_rc_pingpong examples/ibv_uc_pingpong \ - examples/ibv_ud_pingpong examples/ibv_srq_pingpong examples/ibv_xsrq_pingpong -examples_ibv_devices_SOURCES = examples/device_list.c -examples_ibv_devices_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_devinfo_SOURCES = examples/devinfo.c -examples_ibv_devinfo_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_rc_pingpong_SOURCES = examples/rc_pingpong.c examples/pingpong.c -examples_ibv_rc_pingpong_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_uc_pingpong_SOURCES = examples/uc_pingpong.c examples/pingpong.c -examples_ibv_uc_pingpong_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_ud_pingpong_SOURCES = examples/ud_pingpong.c examples/pingpong.c -examples_ibv_ud_pingpong_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_srq_pingpong_SOURCES = examples/srq_pingpong.c examples/pingpong.c -examples_ibv_srq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_xsrq_pingpong_SOURCES = examples/xsrq_pingpong.c examples/pingpong.c -examples_ibv_xsrq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) -examples_ibv_asyncwatch_SOURCES = examples/asyncwatch.c -examples_ibv_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la $(LIBNL_LIBS) - -libibverbsincludedir = $(includedir)/infiniband - -libibverbsinclude_HEADERS = include/infiniband/arch.h include/infiniband/driver.h \ - include/infiniband/kern-abi.h include/infiniband/opcode.h include/infiniband/verbs.h \ - include/infiniband/sa-kern-abi.h include/infiniband/sa.h include/infiniband/marshall.h - -man_MANS = man/ibv_asyncwatch.1 man/ibv_devices.1 man/ibv_devinfo.1 \ - man/ibv_rc_pingpong.1 man/ibv_uc_pingpong.1 man/ibv_ud_pingpong.1 \ - man/ibv_srq_pingpong.1 man/ibv_alloc_pd.3 man/ibv_attach_mcast.3 \ - man/ibv_create_ah.3 man/ibv_create_ah_from_wc.3 \ - man/ibv_create_comp_channel.3 man/ibv_create_cq.3 \ - man/ibv_create_qp.3 man/ibv_create_srq.3 man/ibv_event_type_str.3 \ - man/ibv_fork_init.3 man/ibv_get_async_event.3 man/ibv_create_flow.3 \ - man/ibv_get_cq_event.3 man/ibv_get_device_guid.3 \ - man/ibv_get_device_list.3 man/ibv_get_device_name.3 \ - man/ibv_modify_qp.3 man/ibv_modify_srq.3 man/ibv_open_device.3 \ - man/ibv_poll_cq.3 man/ibv_post_recv.3 man/ibv_post_send.3 \ - man/ibv_post_srq_recv.3 man/ibv_query_device.3 man/ibv_query_gid.3 \ - man/ibv_query_pkey.3 man/ibv_query_port.3 man/ibv_query_qp.3 \ - man/ibv_query_srq.3 man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 \ - man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/ibv_rate_to_mbps.3 \ - man/ibv_create_qp_ex.3 man/ibv_create_srq_ex.3 man/ibv_open_xrcd.3 \ - man/ibv_get_srq_num.3 man/ibv_open_qp.3 man/ibv_query_device_ex.3 \ - man/ibv_alloc_mw.3 man/ibv_bind_mw.3 man/ibv_inc_rkey.3 \ - man/ibv_rereg_mr.3 man/ibv_create_cq_ex.3 \ - man/ibv_query_rt_values_ex.3 man/ibv_xsrq_pingpong.1 \ - man/ibv_create_wq.3 man/ibv_modify_wq.3 \ - man/ibv_create_rwq_ind_table.3 - -DEBIAN = debian/changelog debian/compat debian/control debian/copyright \ - debian/ibverbs-utils.install debian/libibverbs1.install \ - debian/libibverbs1.postinst debian/libibverbs-dev.install \ - debian/rules - -EXTRA_DIST = include/infiniband/driver.h include/infiniband/kern-abi.h \ - include/infiniband/opcode.h include/infiniband/verbs.h include/infiniband/marshall.h \ - include/infiniband/sa-kern-abi.h include/infiniband/sa.h \ - src/ibverbs.h examples/pingpong.h \ - src/libibverbs.map libibverbs.spec.in $(man_MANS) - -dist-hook: libibverbs.spec - cp libibverbs.spec $(distdir) - -install-data-hook: - cd $(DESTDIR)$(mandir)/man3 && \ - $(RM) ibv_ack_async_event.3 && \ - $(RM) ibv_ack_cq_events.3 && \ - $(RM) ibv_close_device.3 && \ - $(RM) ibv_dealloc_pd.3 && \ - $(RM) ibv_dereg_mr.3 && \ - $(RM) ibv_destroy_ah.3 && \ - $(RM) ibv_destroy_comp_channel.3 && \ - $(RM) ibv_destroy_cq.3 && \ - $(RM) ibv_destroy_qp.3 && \ - $(RM) ibv_destroy_srq.3 && \ - $(RM) ibv_detach_mcast.3 && \ - $(RM) ibv_free_device_list.3 && \ - $(RM) ibv_init_ah_from_wc.3 && \ - $(RM) mult_to_ibv_rate.3 && \ - $(RM) ibv_node_type_str.3 && \ - $(RM) ibv_port_state_str.3 && \ - $(RM) mbps_to_ibv_rate.3 && \ - $(RM) ibv_close_xrcd.3 && \ - $(RM) ibv_dealloc_mw.3 && \ - $(RM) ibv_destroy_flow.3 && \ - $(RM) ibv_destroy_wq.3 && \ - $(RM) ibv_destroy_rwq_ind_table.3 && \ - $(LN_S) ibv_get_async_event.3 ibv_ack_async_event.3 && \ - $(LN_S) ibv_get_cq_event.3 ibv_ack_cq_events.3 && \ - $(LN_S) ibv_open_device.3 ibv_close_device.3 && \ - $(LN_S) ibv_alloc_pd.3 ibv_dealloc_pd.3 && \ - $(LN_S) ibv_reg_mr.3 ibv_dereg_mr.3 && \ - $(LN_S) ibv_create_ah.3 ibv_destroy_ah.3 && \ - $(LN_S) ibv_create_comp_channel.3 ibv_destroy_comp_channel.3 && \ - $(LN_S) ibv_create_cq.3 ibv_destroy_cq.3 && \ - $(LN_S) ibv_create_qp.3 ibv_destroy_qp.3 && \ - $(LN_S) ibv_create_srq.3 ibv_destroy_srq.3 && \ - $(LN_S) ibv_attach_mcast.3 ibv_detach_mcast.3 && \ - $(LN_S) ibv_get_device_list.3 ibv_free_device_list.3 && \ - $(LN_S) ibv_create_ah_from_wc.3 ibv_init_ah_from_wc.3 && \ - $(LN_S) ibv_rate_to_mult.3 mult_to_ibv_rate.3 && \ - $(LN_S) ibv_event_type_str.3 ibv_node_type_str.3 && \ - $(LN_S) ibv_event_type_str.3 ibv_port_state_str.3 && \ - $(LN_S) ibv_rate_to_mbps.3 mbps_to_ibv_rate.3 && \ - $(LN_S) ibv_open_xrcd.3 ibv_close_xrcd.3 && \ - $(LN_S) ibv_alloc_mw.3 ibv_dealloc_mw.3 && \ - $(LN_S) ibv_create_flow.3 ibv_destroy_flow.3 && \ - $(LN_S) ibv_create_wq.3 ibv_destroy_wq.3 && \ - $(LN_S) ibv_create_rwq_ind_table.3 ibv_destroy_rwq_ind_table.3 @@ -1,164 +1,15 @@ -Introduction -============ +The libibverbs git repo is no longer in use. The libibverbs package, +as well as all other RDMA related packages that directly open and use +the various kernel device files specific to the RDMA subsystem, have +been pulled together into a single package called rdma-core that is +now hosted on github: -libibverbs is a library that allows programs to use RDMA "verbs" for -direct access to RDMA (currently InfiniBand and iWARP) hardware from -userspace. For more information on RDMA verbs, see the InfiniBand -Architecture Specification vol. 1, especially chapter 11, and the RDMA -Consortium's RDMA Protocol Verbs Specification. +git://github.com/linux-rdma/rdma-core -Using libibverbs -================ +All future code and releases will be through signed tags in the above +repository. -Device nodes ------------- +Thanks, -The verbs library expects special character device files named -/dev/infiniband/uverbsN to be created. When you load the kernel -modules, including both the low-level driver for your IB hardware as -well as the ib_uverbs module, you should see one or more uverbsN -entries in /sys/class/infiniband_verbs in addition to the -/dev/infiniband/uverbsN character device files. +The linux RDMA community. -To create the appropriate character device files automatically with -udev, a rule like - - KERNEL="uverbs*", NAME="infiniband/%k" - -can be used. This will create device nodes named - - /dev/infiniband/uverbs0 - -and so on. Since the RDMA userspace verbs should be safe for use by -non-privileged users, you may want to add an appropriate MODE or GROUP -to your udev rule. - -Permissions ------------ - -To use IB verbs from userspace, a process must be able to access the -appropriate /dev/infiniband/uverbsN special device file. You can -check the permissions on this file with the command - - ls -l /dev/infiniband/uverbs* - -Make sure that the permissions on these files are such that the -user/group that your verbs program runs as can access the device file. - -To use IB verbs from userspace, a process must also have permission to -tell the kernel to lock sufficient memory for all of your registered -memory regions as well as the memory used internally by IB resources -such as queue pairs (QPs) and completion queues (CQs). To check your -resource limits, use the command - - ulimit -l - -(or "limit memorylocked" for csh-like shells). - -If you see a small number such as 32 (the units are KB) then you will -need to increase this limit. This is usually done for ordinary users -via the file /etc/security/limits.conf. More configuration may be -necessary if you are logging in via OpenSSH and your sshd is -configured to use privilege separation. - -Valgrind support ----------------- - -When running applications that use libibverbs under the Valgrind -memory-checking debugger, Valgrind will falsely report "read from -uninitialized" for memory that was initialized by the kernel drivers. -Specifically, Valgrind cannot see when kernel drivers write to -userspace memory, so when the process reads from that memory, Valgrind -incorrectly assumes that the memory contents are uninitialized, and -therefore raises a warning. - -libibverbs can be built with specific support for the Valgrind -memory-checking debugger by specifying the --with-valgrind command -line argument to configure. This flag enables code in libibverbs to -tell Valgrind "this memory may look uninitialized, but it's really -OK," which therefore suppresses the incorrect "read from -uninitialized" warnings. This code adds trivial overhead to the -critical performance path, so it is disabled by default. The intent -is that production users can use a "normal" build of libibverbs and -developers can use the "valgrind debug" build by simply switching -their LD_LIBRARY_PATH environment variables. - -Libibverbs needs some header files from Valgrind in order to compile -this support; it is important to use the header files from the same -version of Valgrind that will be used at run time. You may need to -specify the directory where Valgrind's header files are installed as -an argument to --with-valgrind. For example - - ./configure --with-valgrind=/opt/valgrind - -will make the libibverbs build look for valgrind headers in -/opt/valgrind/include - -Reporting bugs -============== - -Bugs should be reported to the OpenFabrics mailing list -<general@lists.openfabrics.org>. In your bug report, please include: - - * Information about your system: - - Linux distribution and version - - Linux kernel and version - - InfiniBand/iWARP hardware and firmware version - - ... any other relevant information - - * How to reproduce the bug. Command line arguments for a libibverbs - example program or source code that other developers can - compile and run is most convenient. - - * If the bug is a crash, the exact output printed out when the crash - occurred, including any kernel messages produced. - - * If a verbs call is mysteriously returning an error or failing, the - output of "strace -ewrite -ewrite=all <command>". - -Submitting patches -================== - -Patches should also be submitted to the OpenFabrics mailing list -<general@lists.openfabrics.org>. Please use unified diff form (the -u -option to GNU diff), and include a good description of what your patch -does and why it should be applied. If your patch fixes a bug, please -make sure to describe the bug and how your fix works. - -Please include a change to the ChangeLog file (in standard GNU -changelog format) as part of your patch. - -Make sure that your contribution can be licensed under the same -license as the original code you are patching, and that you have all -necessary permissions to release your work. - -TODO -==== - -1.1 series ----------- - -The libibverbs API and ABI are frozen for all releases in the 1.1 -series. Methods were added to struct ibv_context to implement the -following features, so it should be possible to add them in a future -release in the 1.1 series: - - * Memory window (MW) support. - - * Implement the reregister memory region (MR) verb. We will add an - extension to the IB spec to allow the application to indicate that - the region is only being extended, and that operations in progress - should _not_ fail (contrary to the IB spec, which states that - reregister must be implemented so that it behaves equivalently to a - deregister followed by a register). - -Other possibilities -------------------- - -There are no plans to implement the following features, which would be -needed for completeness but don't seem particularly useful. However, -if there is demand from application developers or an implementation is -contributed, then the feature may be added. - - * Implement the query address handle (AH) verb. - * Implement the query memory region (MR) verb. diff --git a/autogen.sh b/autogen.sh deleted file mode 100755 index 6c9233e..0000000 --- a/autogen.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -set -x -autoreconf -ifv -I config diff --git a/config/.gitignore b/config/.gitignore deleted file mode 100644 index 4d4c7b1..0000000 --- a/config/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -mkinstalldirs -depcomp -compile -missing -config.guess -config.sub -ltmain.sh -install-sh diff --git a/configure.ac b/configure.ac deleted file mode 100644 index 90c33df..0000000 --- a/configure.ac +++ /dev/null @@ -1,125 +0,0 @@ -dnl Process this file with autoconf to produce a configure script. - -AC_PREREQ([2.67]) -AC_INIT([libibverbs],[1.2.1],[linux-rdma@vger.kernel.org]) -AC_CONFIG_SRCDIR([src/ibverbs.h]) -AC_CONFIG_AUX_DIR(config) -AC_CONFIG_MACRO_DIR(config) -AC_CONFIG_HEADER(config.h) -AM_INIT_AUTOMAKE([foreign subdir-objects]) -m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) - -dnl Checks for programs -AC_PROG_CC -AC_USE_SYSTEM_EXTENSIONS -AC_PROG_LN_S -LT_INIT - -AC_ARG_WITH([valgrind], - AS_HELP_STRING([--with-valgrind], - [Enable Valgrind annotations (small runtime overhead, default NO)])) -if test x$with_valgrind = x || test x$with_valgrind = xno; then - want_valgrind=no - AC_DEFINE([NVALGRIND], 1, [Define to 1 to disable Valgrind annotations.]) -else - want_valgrind=yes - if test -d $with_valgrind; then - CPPFLAGS="$CPPFLAGS -I$with_valgrind/include" - fi -fi - -AC_ARG_WITH([resolve-neigh], - AC_HELP_STRING([--with-resolve-neigh], - [Enable neighbour resolution in Ethernet (default YES)])) -have_libnl=no -if test x$with_resolve_neigh = x || test x$with_resolve_neigh = xyes; then - PKG_CHECK_MODULES([LIBNL],[libnl-3.0],[ - have_libnl=yes - AC_DEFINE([HAVE_LIBNL3], [1], [Use libnl-3.0]) - AC_DEFINE([HAVE_LIBNL], [1], [Use libnl]) - PKG_CHECK_MODULES([LIBNL_ROUTE3], [libnl-route-3.0]) - LIBNL_CFLAGS="$LIBNL_CFLAGS $LIBNL_ROUTE3_CFLAGS" - LIBNL_LIBS="$LIBNL_LIBS $LIBNL_ROUTE3_LIBS"], [:] - ); - if test "$have_libnl" = no; then - PKG_CHECK_MODULES([LIBNL], [libnl-1], [have_libnl=yes - AC_DEFINE([HAVE_LIBNL1], [1], [Use libnl-1]) - AC_DEFINE([HAVE_LIBNL], [1], [Use libnl]) - AC_CHECK_LIB(nl, rtnl_link_vlan_get_id, [], - AC_MSG_ERROR([rtnl_link_vlan_get_id not found. libibverbs requires libnl.])) - ],[ - AC_MSG_ERROR([libibverbs requires libnl.]) - ]) - fi -else - AC_DEFINE([NRESOLVE_NEIGH], 1, [Define to 1 to disable resovle neigh annotations.]) -fi -AM_CONDITIONAL([HAVE_LIBNL], [test "$have_libnl" = "yes"]) -AC_SUBST([LIBNL_CFLAGS]) -AC_SUBST([LIBNL_LIBS]) -AM_CONDITIONAL(NO_RESOLVE_NEIGH, test x$with_resolve_neigh = xno) - -dnl Checks for libraries -AC_CHECK_LIB(dl, dlsym, [], - AC_MSG_ERROR([dlsym() not found. libibverbs requires libdl.])) -AC_CHECK_LIB(pthread, pthread_mutex_init, [], - AC_MSG_ERROR([pthread_mutex_init() not found. libibverbs requires libpthread.])) - -dnl Checks for header files. -AC_HEADER_STDC -AC_CHECK_HEADER(valgrind/memcheck.h, - [AC_DEFINE(HAVE_VALGRIND_MEMCHECK_H, 1, - [Define to 1 if you have the <valgrind/memcheck.h> header file.])], - [if test $want_valgrind = yes; then - AC_MSG_ERROR([Valgrind memcheck support requested, but <valgrind/memcheck.h> not found.]) - fi]) -AC_CHECK_HEADERS([fcntl.h sys/socket.h]) - -dnl Checks for typedefs, structures, and compiler characteristics. -AC_C_CONST -AC_CHECK_DECLS([O_CLOEXEC],,[AC_DEFINE([O_CLOEXEC],[0], [Defined to 0 if not provided])], -[[ -#ifdef HAVE_FCNTL_H -# include <fcntl.h> -#endif -]]) -AC_CHECK_DECLS([SOCK_CLOEXEC],,[AC_DEFINE([SOCK_CLOEXEC],[0],[Defined to 0 if not provided])], -[[ -#ifdef HAVE_SYS_SOCKET_H -# include <sys/socket.h> -#endif -]]) - -AC_CACHE_CHECK(for close on exec modifier for fopen(), ac_cv_feature_stream_cloexec_flag, - [if test $ac_cv_have_decl_O_CLOEXEC = yes ; then - if test $ac_cv_have_decl_SOCK_CLOEXEC = yes ; then - ac_cv_feature_stream_cloexec_flag="e" - fi - fi]) -AC_DEFINE_UNQUOTED([STREAM_CLOEXEC], "$ac_cv_feature_stream_cloexec_flag", [fopen() modifier for setting close on exec flag]) - -AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script, - [if test -n "`$LD --help < /dev/null 2>/dev/null | grep version-script`"; then - ac_cv_version_script=yes - else - ac_cv_version_script=no - fi]) - -if test $ac_cv_version_script = yes; then - LIBIBVERBS_VERSION_SCRIPT='-Wl,--version-script=$(srcdir)/src/libibverbs.map' -else - LIBIBVERBS_VERSION_SCRIPT= -fi -AC_SUBST(LIBIBVERBS_VERSION_SCRIPT) - -AC_CACHE_CHECK(for .symver assembler support, ac_cv_asm_symver_support, - [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], - [[asm("symbol:\n.symver symbol, api@ABI\n");]])], - [ac_cv_asm_symver_support=yes], - [ac_cv_asm_symver_support=no])]) -if test $ac_cv_asm_symver_support = yes; then - AC_DEFINE([HAVE_SYMVER_SUPPORT], 1, [assembler has .symver support]) -fi - -AC_CONFIG_FILES([Makefile libibverbs.spec]) -AC_OUTPUT diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 3ae7711..0000000 --- a/debian/changelog +++ /dev/null @@ -1,139 +0,0 @@ -libibverbs (1.1.8-1) unstable; urgency=low - - * New upstream release. - - XRC support - - Flow steering support - - Generic infrastructure for further "verbs extensions" - * Update to Standards-Version: 3.9.5. - - -- Roland Dreier <rbd@debian.org> Mon, 05 May 2014 09:50:44 -0700 - -libibverbs (1.1.7-1) unstable; urgency=low - - * New upstream release. - - Add raw QP support - - Further support for new IB link speeds - - Other miscellaneous fixes - * Update maintainer and remove DM-Upload-Allowed now that I'm a DD. - - -- Roland Dreier <rbd@debian.org> Wed, 29 Jan 2014 13:02:21 -0800 - -libibverbs (1.1.6-1) unstable; urgency=low - - * Don't use brace expansion for {a,so} in libibverbs-dev.install - * New upstream releases. - - Add IBoE support - - Add ibv_devinfo support for displaying extended speeds (FDR, etc) - - -- Roland Dreier <roland@digitalvampire.org> Wed, 21 Dec 2011 16:35:08 -0800 - -libibverbs (1.1.5-1) unstable; urgency=low - - * Switch to dpkg-source 3.0 (quilt) format. - * New upstream releases. - - Fix handling of madvise() failures - - Add path record definitions to sa.h - - Handle huge pages for fork support/madvise - - Fix crash if no devices and ibv_get_device_list() called multiple times - * Update maintainer and set DM-Upload-Allowed to yes. (Closes: #632106) - * Don't ship .la files. - * Update to Standards-Version: 3.9.2. - - -- Roland Dreier <roland@digitalvampire.org> Wed, 29 Jun 2011 23:57:34 -0700 - -libibverbs (1.1.3-2) unstable; urgency=low - - * Add override_dh_strip target to get debugging information actually - put into the -dbg package. - - -- Roland Dreier <rolandd@cisco.com> Wed, 11 Nov 2009 14:44:51 -0800 - -libibverbs (1.1.3-1) unstable; urgency=low - - * New upstream release. - - Don't use enums for bit flags (avoid C++ annoyances) - - Return some errors via errno instead of stderr - * Move -dbg package to section debug. - * Update to Standards-Version: 3.8.3 (no changes needed). - * Change build system from cdbs to debhelper 7. - - -- Roland Dreier <rolandd@cisco.com> Thu, 29 Oct 2009 14:19:02 -0700 - -libibverbs (1.1.2-1) unstable; urgency=low - - * New upstream release. - - Fix memory registration failure cause by too-big madvise() - - Fix many Valgrind false positives - - Add functions to convert enum values to strings - * Replace deprecated ${Source-Version} with ${binary:Version} - * Use DEB_DH_MAKESHLIBS_ARGS_ALL to pass appropriate -V option to - dh_makeshlibs, since new symbols were added in libibverbs 1.1.2. - (Closes: #465435) - * Add debian/watch file. - * Update control file to talk about generic RDMA and iWARP, not just - InfiniBand, since libibverbs works with both IB and iWARP. - * Acknowledge NMU (Closes: #442638). - - -- Roland Dreier <rolandd@cisco.com> Fri, 18 Apr 2008 15:08:52 -0700 - -libibverbs (1.1.1-1.1) unstable; urgency=low - - * Non-maintainer upload. - * Re-generated autotools files to fix double build bug, closes: #442638 - * Bumped Standards-Version to 3.7.3, no change needed. - - -- Michael Meskes <meskes@debian.org> Mon, 14 Apr 2008 10:07:58 +0000 - -libibverbs (1.1.1-1) unstable; urgency=low - - * New upstream release. - - Initialize state of newly created QPs to RESET (fixes problems - with libmlx4/ConnectX HCAs). - - Don't warn root about RLIMIT_MEMLOCK, since it doesn't matter. - - Fix free() errors in ibv_xx_pingpong examples. - - -- Roland Dreier <rolandd@cisco.com> Fri, 15 Jun 2007 12:49:02 -0700 - -libibverbs (1.1-1) unstable; urgency=low - - * New upstream release. - - Add support for use of fork() in applications. - - Add manual pages documenting API in section 3. - - New method of finding and loading device-specific drivers. - - Add basic support for iWARP devices. - - Provide compatible ABI for applications linked against libibverbs 1.0. - * Update libtool during build to avoid setting RPATH in binaries on amd64. - - -- Roland Dreier <rolandd@cisco.com> Sat, 28 Apr 2007 14:15:29 -0700 - -libibverbs (1.0.4-1) unstable; urgency=low - - * New upstream release. - - Fix static linking so it has a chance of working. - - Fix cut-and-paste error in sparc mb() macro. - - Other miscellaneous fixes. - * Improve package description. - - -- Roland Dreier <rolandd@cisco.com> Tue, 31 Oct 2006 15:04:33 -0800 - -libibverbs (1.0.3-1) unstable; urgency=low - - * Change priority to extra, since libibverbs depends on libsysfs2, which - has priority extra. (Debian policy section 2.5 states that a package - may not depend on another package of lower priority) - * New upstream release: - - For sparc, only generate membar instruction if compiling for V9 - instruction set. (Closes: #365559) - - Reduce (but not yet eliminate) dependency on libsysfs. - - Deprecate some ib_XXX symbol names and introduce ibv_XXX - replacements for internal consistency. - - Other minor fixes. - * Update to Standards-Version: 3.7.2. - - -- Roland Dreier <rolandd@cisco.com> Tue, 2 May 2006 15:33:14 -0700 - -libibverbs (1.0.2-1) unstable; urgency=low - - * Initial Release. (Closes: #325752) - - -- Roland Dreier <rolandd@cisco.com> Wed, 15 Feb 2006 11:21:59 -0700 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7f8f011..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/debian/control b/debian/control deleted file mode 100644 index a26b75c..0000000 --- a/debian/control +++ /dev/null @@ -1,80 +0,0 @@ -Source: libibverbs -Priority: extra -Maintainer: Roland Dreier <rbd@debian.org> -Build-Depends: debhelper (>= 7.0.50~), dpkg-dev (>= 1.13.19) -Standards-Version: 3.9.5 -Section: libs -Homepage: http://www.openfabrics.org/ - -Package: libibverbs1 -Section: libs -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, adduser -Description: Library for direct userspace use of RDMA (InfiniBand/iWARP) - libibverbs is a library that allows userspace processes to use RDMA - "verbs" as described in the InfiniBand Architecture Specification and - the RDMA Protocol Verbs Specification. iWARP ethernet NICs support - RDMA over hardware-offloaded TCP/IP, while InfiniBand is a - high-throughput, low-latency networking technology. InfiniBand host - channel adapters (HCAs) and iWARP NICs commonly support direct - hardware access from userspace (kernel bypass), and libibverbs - supports this when available. - . - For this library to be useful, a device-specific plug-in module - should also be installed. - . - This package contains the shared library. - -Package: libibverbs-dev -Section: libdevel -Architecture: any -Depends: ${misc:Depends}, libibverbs1 (= ${binary:Version}) -Description: Development files for the libibverbs library - libibverbs is a library that allows userspace processes to use RDMA - "verbs" as described in the InfiniBand Architecture Specification and - the RDMA Protocol Verbs Specification. iWARP ethernet NICs support - RDMA over hardware-offloaded TCP/IP, while InfiniBand is a - high-throughput, low-latency networking technology. InfiniBand host - channel adapters (HCAs) and iWARP NICs commonly support direct - hardware access from userspace (kernel bypass), and libibverbs - supports this when available. - . - This package is needed to compile programs against libibverbs1. - It contains the header files and static libraries (optionally) - needed for compiling. - -Package: libibverbs1-dbg -Section: debug -Priority: extra -Architecture: any -Depends: ${misc:Depends}, libibverbs1 (= ${binary:Version}) -Description: Debugging symbols for the libibverbs library - libibverbs is a library that allows userspace processes to use RDMA - "verbs" as described in the InfiniBand Architecture Specification and - the RDMA Protocol Verbs Specification. iWARP ethernet NICs support - RDMA over hardware-offloaded TCP/IP, while InfiniBand is a - high-throughput, low-latency networking technology. InfiniBand host - channel adapters (HCAs) and iWARP NICs commonly support direct - hardware access from userspace (kernel bypass), and libibverbs - supports this when available. - . - This package contains the debugging symbols associated with - libibverbs1. They will automatically be used by gdb for debugging - libibverbs-related issues. - -Package: ibverbs-utils -Section: net -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends} -Description: Examples for the libibverbs library - libibverbs is a library that allows userspace processes to use RDMA - "verbs" as described in the InfiniBand Architecture Specification and - the RDMA Protocol Verbs Specification. iWARP ethernet NICs support - RDMA over hardware-offloaded TCP/IP, while InfiniBand is a - high-throughput, low-latency networking technology. InfiniBand host - channel adapters (HCAs) and iWARP NICs commonly support direct - hardware access from userspace (kernel bypass), and libibverbs - supports this when available. - . - This package contains useful libibverbs1 example programs such as - ibv_devinfo, which displays information about InfiniBand devices. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 5009e7a..0000000 --- a/debian/copyright +++ /dev/null @@ -1,49 +0,0 @@ -Initial Debianization: -This package was debianized by Roland Dreier <rolandd@cisco.com> on -Mon, 25 Apr 2005 10:21:08 -0700. - -Source: -It was downloaded from the OpenIB web site at -<https://openib.org/downloads.html> - -Authors: - Roland Dreier <roland@topspin.com> - Dotan Barak <dotanba@gmail.com> - Sean Hefty <sean.hefty@intel.com> - Michael S. Tsirkin <mst@mellanox.co.il> - -Portions are copyrighted by: - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. - * Copyright (c) 2005 PathScale, Inc. All rights reserved. - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Voltaire, Inc. All rights reserved. - -libibverbs is licensed under a choice of one of two licenses. You may -choose to be licensed under the terms of the GNU General Public -License (GPL) Version 2, available from the file -/usr/share/common-licenses/GPL-2 on your Debian system, or the -OpenIB.org BSD license below: - - Redistribution and use in source and binary forms, with or - without modification, are permitted provided that the following - conditions are met: - - - Redistributions of source code must retain the above - copyright notice, this list of conditions and the following - disclaimer. - - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/debian/ibverbs-utils.install b/debian/ibverbs-utils.install deleted file mode 100644 index 98d1583..0000000 --- a/debian/ibverbs-utils.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/bin -usr/share/man/man1 diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install deleted file mode 100644 index b4c75ce..0000000 --- a/debian/libibverbs-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/include -usr/lib/libibverbs*.a -usr/lib/libibverbs*.so -usr/share/man/man3 diff --git a/debian/libibverbs-dev.links b/debian/libibverbs-dev.links deleted file mode 100644 index 7029028..0000000 --- a/debian/libibverbs-dev.links +++ /dev/null @@ -1,16 +0,0 @@ -usr/share/man/man3/ibv_get_async_event.3 usr/share/man/man3/ibv_ack_async_event.3 -usr/share/man/man3/ibv_get_cq_event.3 usr/share/man/man3/ibv_ack_cq_events.3 -usr/share/man/man3/ibv_open_device.3 usr/share/man/man3/ibv_close_device.3 -usr/share/man/man3/ibv_alloc_pd.3 usr/share/man/man3/ibv_dealloc_pd.3 -usr/share/man/man3/ibv_reg_mr.3 usr/share/man/man3/ibv_dereg_mr.3 -usr/share/man/man3/ibv_create_ah.3 usr/share/man/man3/ibv_destroy_ah.3 -usr/share/man/man3/ibv_create_comp_channel.3 usr/share/man/man3/ibv_destroy_comp_channel.3 -usr/share/man/man3/ibv_create_cq.3 usr/share/man/man3/ibv_destroy_cq.3 -usr/share/man/man3/ibv_create_qp.3 usr/share/man/man3/ibv_destroy_qp.3 -usr/share/man/man3/ibv_create_srq.3 usr/share/man/man3/ibv_destroy_srq.3 -usr/share/man/man3/ibv_attach_mcast.3 usr/share/man/man3/ibv_detach_mcast.3 -usr/share/man/man3/ibv_get_device_list.3 usr/share/man/man3/ibv_free_device_list.3 -usr/share/man/man3/ibv_create_ah_from_wc.3 usr/share/man/man3/ibv_init_ah_from_wc.3 -usr/share/man/man3/ibv_rate_to_mult.3 usr/share/man/man3/mult_to_ibv_rate.3 -usr/share/man/man3/ibv_event_type_str.3 usr/share/man/man3/ibv_node_type_str.3 -usr/share/man/man3/ibv_event_type_str.3 usr/share/man/man3/ibv_port_state_str.3 diff --git a/debian/libibverbs1.install b/debian/libibverbs1.install deleted file mode 100644 index 0f3523d..0000000 --- a/debian/libibverbs1.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/libibverbs*.so.* diff --git a/debian/libibverbs1.postinst b/debian/libibverbs1.postinst deleted file mode 100644 index 784458a..0000000 --- a/debian/libibverbs1.postinst +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# postinst script for libibverbs1 - -set -e - -if [ "$1" = configure ]; then - getent group rdma > /dev/null 2>&1 || addgroup --system --quiet rdma -fi - -#DEBHELPER# diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 205a8b7..0000000 --- a/debian/rules +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/make -f -# -*- mode: makefile; coding: utf-8 -*- - -%: - dh $@ - -override_dh_strip: - dh_strip --dbg-package=libibverbs1-dbg - -override_dh_makeshlibs: - dh_makeshlibs -V 'libibverbs1 (>= 1.1.2)' diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/debian/watch b/debian/watch deleted file mode 100644 index eff271a..0000000 --- a/debian/watch +++ /dev/null @@ -1,3 +0,0 @@ -version=3 -opts="uversionmangle=s/-rc/~rc/" \ - http://www.openfabrics.org/downloads/verbs/libibverbs-(.+)\.tar\.gz diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index ebecbdc..0000000 --- a/examples/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.dirstamp -ibv_asyncwatch -ibv_devices -ibv_devinfo -ibv_rc_pingpong -ibv_srq_pingpong -ibv_uc_pingpong -ibv_ud_pingpong -ibv_xsrq_pingpong -.libs diff --git a/examples/asyncwatch.c b/examples/asyncwatch.c deleted file mode 100644 index a77c1c8..0000000 --- a/examples/asyncwatch.c +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <endian.h> -#include <byteswap.h> -#include <getopt.h> -#include <string.h> - -#include <infiniband/verbs.h> - -static const char *event_name_str(enum ibv_event_type event_type) -{ - switch (event_type) { - case IBV_EVENT_DEVICE_FATAL: - return "IBV_EVENT_DEVICE_FATAL"; - case IBV_EVENT_PORT_ACTIVE: - return "IBV_EVENT_PORT_ACTIVE"; - case IBV_EVENT_PORT_ERR: - return "IBV_EVENT_PORT_ERR"; - case IBV_EVENT_LID_CHANGE: - return "IBV_EVENT_LID_CHANGE"; - case IBV_EVENT_PKEY_CHANGE: - return "IBV_EVENT_PKEY_CHANGE"; - case IBV_EVENT_SM_CHANGE: - return "IBV_EVENT_SM_CHANGE"; - case IBV_EVENT_CLIENT_REREGISTER: - return "IBV_EVENT_CLIENT_REREGISTER"; - case IBV_EVENT_GID_CHANGE: - return "IBV_EVENT_GID_CHANGE"; - - case IBV_EVENT_CQ_ERR: - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_COMM_EST: - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - case IBV_EVENT_QP_LAST_WQE_REACHED: - default: - return "unexpected"; - } -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start an asyncwatch process\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -h, --help print a help text and exit\n"); -} - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - struct ibv_context *context; - struct ibv_async_event event; - char *ib_devname = NULL; - int i = 0; - - /* Force line-buffering in case stdout is redirected */ - setvbuf(stdout, NULL, _IOLBF, 0); - - while (1) { - int ret = 1; - int c; - static struct option long_options[] = { - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "help", .has_arg = 0, .val = 'h' }, - { 0 } - }; - - c = getopt_long(argc, argv, "d:h", long_options, NULL); - if (c == -1) - break; - switch (c) { - case 'd': - ib_devname = strdupa(optarg); - break; - case 'h': - ret = 0; - default: - usage(argv[0]); - return ret; - } - } - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - if (ib_devname) { - for (; dev_list[i]; ++i) { - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - } - } - - if (!dev_list[i]) { - fprintf(stderr, "IB device %s not found\n", - ib_devname ? ib_devname : ""); - return 1; - } - - context = ibv_open_device(dev_list[i]); - if (!context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(dev_list[i])); - return 1; - } - - printf("%s: async event FD %d\n", - ibv_get_device_name(dev_list[i]), context->async_fd); - - while (1) { - if (ibv_get_async_event(context, &event)) - return 1; - - printf(" event_type %s (%d), port %d\n", - event_name_str(event.event_type), - event.event_type, event.element.port_num); - - ibv_ack_async_event(&event); - } - - return 0; -} diff --git a/examples/device_list.c b/examples/device_list.c deleted file mode 100644 index 70c3af5..0000000 --- a/examples/device_list.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> - -#include <endian.h> -#include <byteswap.h> - -#include <infiniband/verbs.h> -#include <infiniband/arch.h> - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - int num_devices, i; - - dev_list = ibv_get_device_list(&num_devices); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - - printf(" %-16s\t node GUID\n", "device"); - printf(" %-16s\t----------------\n", "------"); - - for (i = 0; i < num_devices; ++i) { - printf(" %-16s\t%016llx\n", - ibv_get_device_name(dev_list[i]), - (unsigned long long) ntohll(ibv_get_device_guid(dev_list[i]))); - } - - ibv_free_device_list(dev_list); - - return 0; -} diff --git a/examples/devinfo.c b/examples/devinfo.c deleted file mode 100644 index 1d0c2db..0000000 --- a/examples/devinfo.c +++ /dev/null @@ -1,666 +0,0 @@ -/* - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <getopt.h> -#include <netinet/in.h> -#include <endian.h> -#include <byteswap.h> -#include <inttypes.h> - -#include <infiniband/verbs.h> -#include <infiniband/driver.h> -#include <infiniband/arch.h> - -static int verbose; - -static int null_gid(union ibv_gid *gid) -{ - return !(gid->raw[8] | gid->raw[9] | gid->raw[10] | gid->raw[11] | - gid->raw[12] | gid->raw[13] | gid->raw[14] | gid->raw[15]); -} - -static const char *guid_str(uint64_t node_guid, char *str) -{ - node_guid = ntohll(node_guid); - sprintf(str, "%04x:%04x:%04x:%04x", - (unsigned) (node_guid >> 48) & 0xffff, - (unsigned) (node_guid >> 32) & 0xffff, - (unsigned) (node_guid >> 16) & 0xffff, - (unsigned) (node_guid >> 0) & 0xffff); - return str; -} - -static const char *transport_str(enum ibv_transport_type transport) -{ - switch (transport) { - case IBV_TRANSPORT_IB: return "InfiniBand"; - case IBV_TRANSPORT_IWARP: return "iWARP"; - case IBV_TRANSPORT_USNIC: return "usNIC"; - case IBV_TRANSPORT_USNIC_UDP: return "usNIC UDP"; - default: return "invalid transport"; - } -} - -static const char *port_state_str(enum ibv_port_state pstate) -{ - switch (pstate) { - case IBV_PORT_DOWN: return "PORT_DOWN"; - case IBV_PORT_INIT: return "PORT_INIT"; - case IBV_PORT_ARMED: return "PORT_ARMED"; - case IBV_PORT_ACTIVE: return "PORT_ACTIVE"; - default: return "invalid state"; - } -} - -static const char *port_phy_state_str(uint8_t phys_state) -{ - switch (phys_state) { - case 1: return "SLEEP"; - case 2: return "POLLING"; - case 3: return "DISABLED"; - case 4: return "PORT_CONFIGURATION TRAINNING"; - case 5: return "LINK_UP"; - case 6: return "LINK_ERROR_RECOVERY"; - case 7: return "PHY TEST"; - default: return "invalid physical state"; - } -} - -static const char *atomic_cap_str(enum ibv_atomic_cap atom_cap) -{ - switch (atom_cap) { - case IBV_ATOMIC_NONE: return "ATOMIC_NONE"; - case IBV_ATOMIC_HCA: return "ATOMIC_HCA"; - case IBV_ATOMIC_GLOB: return "ATOMIC_GLOB"; - default: return "invalid atomic capability"; - } -} - -static const char *mtu_str(enum ibv_mtu max_mtu) -{ - switch (max_mtu) { - case IBV_MTU_256: return "256"; - case IBV_MTU_512: return "512"; - case IBV_MTU_1024: return "1024"; - case IBV_MTU_2048: return "2048"; - case IBV_MTU_4096: return "4096"; - default: return "invalid MTU"; - } -} - -static const char *width_str(uint8_t width) -{ - switch (width) { - case 1: return "1"; - case 2: return "4"; - case 4: return "8"; - case 8: return "12"; - default: return "invalid width"; - } -} - -static const char *speed_str(uint8_t speed) -{ - switch (speed) { - case 1: return "2.5 Gbps"; - case 2: return "5.0 Gbps"; - - case 4: /* fall through */ - case 8: return "10.0 Gbps"; - - case 16: return "14.0 Gbps"; - case 32: return "25.0 Gbps"; - default: return "invalid speed"; - } -} - -static const char *vl_str(uint8_t vl_num) -{ - switch (vl_num) { - case 1: return "1"; - case 2: return "2"; - case 3: return "4"; - case 4: return "8"; - case 5: return "15"; - default: return "invalid value"; - } -} - -static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tbl_len) -{ - union ibv_gid gid; - int rc = 0; - int i; - - for (i = 0; i < tbl_len; i++) { - rc = ibv_query_gid(ctx, port_num, i, &gid); - if (rc) { - fprintf(stderr, "Failed to query gid to port %d, index %d\n", - port_num, i); - return rc; - } - if (!null_gid(&gid)) - printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n", - i, - gid.raw[ 0], gid.raw[ 1], - gid.raw[ 2], gid.raw[ 3], - gid.raw[ 4], gid.raw[ 5], - gid.raw[ 6], gid.raw[ 7], - gid.raw[ 8], gid.raw[ 9], - gid.raw[10], gid.raw[11], - gid.raw[12], gid.raw[13], - gid.raw[14], gid.raw[15]); - } - return rc; -} - -static const char *link_layer_str(uint8_t link_layer) -{ - switch (link_layer) { - case IBV_LINK_LAYER_UNSPECIFIED: - case IBV_LINK_LAYER_INFINIBAND: - return "InfiniBand"; - case IBV_LINK_LAYER_ETHERNET: - return "Ethernet"; - default: - return "Unknown"; - } -} - -static void print_device_cap_flags(uint32_t dev_cap_flags) -{ - uint32_t unknown_flags = ~(IBV_DEVICE_RESIZE_MAX_WR | - IBV_DEVICE_BAD_PKEY_CNTR | - IBV_DEVICE_BAD_QKEY_CNTR | - IBV_DEVICE_RAW_MULTI | - IBV_DEVICE_AUTO_PATH_MIG | - IBV_DEVICE_CHANGE_PHY_PORT | - IBV_DEVICE_UD_AV_PORT_ENFORCE | - IBV_DEVICE_CURR_QP_STATE_MOD | - IBV_DEVICE_SHUTDOWN_PORT | - IBV_DEVICE_INIT_TYPE | - IBV_DEVICE_PORT_ACTIVE_EVENT | - IBV_DEVICE_SYS_IMAGE_GUID | - IBV_DEVICE_RC_RNR_NAK_GEN | - IBV_DEVICE_SRQ_RESIZE | - IBV_DEVICE_N_NOTIFY_CQ | - IBV_DEVICE_MEM_WINDOW | - IBV_DEVICE_UD_IP_CSUM | - IBV_DEVICE_XRC | - IBV_DEVICE_MEM_MGT_EXTENSIONS | - IBV_DEVICE_MEM_WINDOW_TYPE_2A | - IBV_DEVICE_MEM_WINDOW_TYPE_2B | - IBV_DEVICE_RC_IP_CSUM | - IBV_DEVICE_RAW_IP_CSUM | - IBV_DEVICE_MANAGED_FLOW_STEERING); - - if (dev_cap_flags & IBV_DEVICE_RESIZE_MAX_WR) - printf("\t\t\t\t\tRESIZE_MAX_WR\n"); - if (dev_cap_flags & IBV_DEVICE_BAD_PKEY_CNTR) - printf("\t\t\t\t\tBAD_PKEY_CNTR\n"); - if (dev_cap_flags & IBV_DEVICE_BAD_QKEY_CNTR) - printf("\t\t\t\t\tBAD_QKEY_CNTR\n"); - if (dev_cap_flags & IBV_DEVICE_RAW_MULTI) - printf("\t\t\t\t\tRAW_MULTI\n"); - if (dev_cap_flags & IBV_DEVICE_AUTO_PATH_MIG) - printf("\t\t\t\t\tAUTO_PATH_MIG\n"); - if (dev_cap_flags & IBV_DEVICE_CHANGE_PHY_PORT) - printf("\t\t\t\t\tCHANGE_PHY_PORT\n"); - if (dev_cap_flags & IBV_DEVICE_UD_AV_PORT_ENFORCE) - printf("\t\t\t\t\tUD_AV_PORT_ENFORCE\n"); - if (dev_cap_flags & IBV_DEVICE_CURR_QP_STATE_MOD) - printf("\t\t\t\t\tCURR_QP_STATE_MOD\n"); - if (dev_cap_flags & IBV_DEVICE_SHUTDOWN_PORT) - printf("\t\t\t\t\tSHUTDOWN_PORT\n"); - if (dev_cap_flags & IBV_DEVICE_INIT_TYPE) - printf("\t\t\t\t\tINIT_TYPE\n"); - if (dev_cap_flags & IBV_DEVICE_PORT_ACTIVE_EVENT) - printf("\t\t\t\t\tPORT_ACTIVE_EVENT\n"); - if (dev_cap_flags & IBV_DEVICE_SYS_IMAGE_GUID) - printf("\t\t\t\t\tSYS_IMAGE_GUID\n"); - if (dev_cap_flags & IBV_DEVICE_RC_RNR_NAK_GEN) - printf("\t\t\t\t\tRC_RNR_NAK_GEN\n"); - if (dev_cap_flags & IBV_DEVICE_SRQ_RESIZE) - printf("\t\t\t\t\tSRQ_RESIZE\n"); - if (dev_cap_flags & IBV_DEVICE_N_NOTIFY_CQ) - printf("\t\t\t\t\tN_NOTIFY_CQ\n"); - if (dev_cap_flags & IBV_DEVICE_MEM_WINDOW) - printf("\t\t\t\t\tMEM_WINDOW\n"); - if (dev_cap_flags & IBV_DEVICE_UD_IP_CSUM) - printf("\t\t\t\t\tUD_IP_CSUM\n"); - if (dev_cap_flags & IBV_DEVICE_XRC) - printf("\t\t\t\t\tXRC\n"); - if (dev_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) - printf("\t\t\t\t\tMEM_MGT_EXTENSIONS\n"); - if (dev_cap_flags & IBV_DEVICE_MEM_WINDOW_TYPE_2A) - printf("\t\t\t\t\tMEM_WINDOW_TYPE_2A\n"); - if (dev_cap_flags & IBV_DEVICE_MEM_WINDOW_TYPE_2B) - printf("\t\t\t\t\tMEM_WINDOW_TYPE_2B\n"); - if (dev_cap_flags & IBV_DEVICE_RC_IP_CSUM) - printf("\t\t\t\t\tRC_IP_CSUM\n"); - if (dev_cap_flags & IBV_DEVICE_RAW_IP_CSUM) - printf("\t\t\t\t\tRAW_IP_CSUM\n"); - if (dev_cap_flags & IBV_DEVICE_MANAGED_FLOW_STEERING) - printf("\t\t\t\t\tMANAGED_FLOW_STEERING\n"); - if (dev_cap_flags & unknown_flags) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX32 "\n", - dev_cap_flags & unknown_flags); -} - -void print_odp_trans_caps(uint32_t trans) -{ - uint32_t unknown_transport_caps = ~(IBV_ODP_SUPPORT_SEND | - IBV_ODP_SUPPORT_RECV | - IBV_ODP_SUPPORT_WRITE | - IBV_ODP_SUPPORT_READ | - IBV_ODP_SUPPORT_ATOMIC); - - if (!trans) { - printf("\t\t\t\t\tNO SUPPORT\n"); - } else { - if (trans & IBV_ODP_SUPPORT_SEND) - printf("\t\t\t\t\tSUPPORT_SEND\n"); - if (trans & IBV_ODP_SUPPORT_RECV) - printf("\t\t\t\t\tSUPPORT_RECV\n"); - if (trans & IBV_ODP_SUPPORT_WRITE) - printf("\t\t\t\t\tSUPPORT_WRITE\n"); - if (trans & IBV_ODP_SUPPORT_READ) - printf("\t\t\t\t\tSUPPORT_READ\n"); - if (trans & IBV_ODP_SUPPORT_ATOMIC) - printf("\t\t\t\t\tSUPPORT_ATOMIC\n"); - if (trans & unknown_transport_caps) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX32 "\n", - trans & unknown_transport_caps); - } -} - -void print_odp_caps(const struct ibv_odp_caps *caps) -{ - uint64_t unknown_general_caps = ~(IBV_ODP_SUPPORT); - - /* general odp caps */ - printf("\tgeneral_odp_caps:\n"); - if (caps->general_caps & IBV_ODP_SUPPORT) - printf("\t\t\t\t\tODP_SUPPORT\n"); - if (caps->general_caps & unknown_general_caps) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX64 "\n", - caps->general_caps & unknown_general_caps); - - /* RC transport */ - printf("\trc_odp_caps:\n"); - print_odp_trans_caps(caps->per_transport_caps.rc_odp_caps); - printf("\tuc_odp_caps:\n"); - print_odp_trans_caps(caps->per_transport_caps.uc_odp_caps); - printf("\tud_odp_caps:\n"); - print_odp_trans_caps(caps->per_transport_caps.ud_odp_caps); -} - -static void print_device_cap_flags_ex(uint64_t device_cap_flags_ex) -{ - uint64_t ex_flags = device_cap_flags_ex & 0xffffffff00000000; - uint64_t unknown_flags = ~(IBV_DEVICE_RAW_SCATTER_FCS); - - if (ex_flags & IBV_DEVICE_RAW_SCATTER_FCS) - printf("\t\t\t\t\tRAW_SCATTER_FCS\n"); - if (ex_flags & unknown_flags) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX64 "\n", - ex_flags & unknown_flags); -} - -static void print_tso_caps(const struct ibv_tso_caps *caps) -{ - uint32_t unknown_general_caps = ~(1 << IBV_QPT_RAW_PACKET | - 1 << IBV_QPT_UD); - printf("\ttso_caps:\n"); - printf("\tmax_tso:\t\t\t%d\n", caps->max_tso); - - if (caps->max_tso) { - printf("\tsupported_qp:\n"); - if (ibv_is_qpt_supported(caps->supported_qpts, IBV_QPT_RAW_PACKET)) - printf("\t\t\t\t\tSUPPORT_RAW_PACKET\n"); - if (ibv_is_qpt_supported(caps->supported_qpts, IBV_QPT_UD)) - printf("\t\t\t\t\tSUPPORT_UD\n"); - if (caps->supported_qpts & unknown_general_caps) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX32 "\n", - caps->supported_qpts & unknown_general_caps); - } -} - -static void print_rss_caps(const struct ibv_rss_caps *caps) -{ - uint32_t unknown_general_caps = ~(1 << IBV_QPT_RAW_PACKET | - 1 << IBV_QPT_UD); - printf("\trss_caps:\n"); - printf("\t\tmax_rwq_indirection_tables:\t\t\t%u\n", caps->max_rwq_indirection_tables); - printf("\t\tmax_rwq_indirection_table_size:\t\t\t%u\n", caps->max_rwq_indirection_table_size); - printf("\t\trx_hash_function:\t\t\t\t0x%x\n", caps->rx_hash_function); - printf("\t\trx_hash_fields_mask:\t\t\t\t0x%" PRIX64 "\n", caps->rx_hash_fields_mask); - - if (caps->supported_qpts) { - printf("\t\tsupported_qp:\n"); - if (ibv_is_qpt_supported(caps->supported_qpts, IBV_QPT_RAW_PACKET)) - printf("\t\t\t\t\tSUPPORT_RAW_PACKET\n"); - if (ibv_is_qpt_supported(caps->supported_qpts, IBV_QPT_UD)) - printf("\t\t\t\t\tSUPPORT_UD\n"); - if (caps->supported_qpts & unknown_general_caps) - printf("\t\t\t\t\tUnknown flags: 0x%" PRIX32 "\n", - caps->supported_qpts & unknown_general_caps); - } -} - -static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) -{ - struct ibv_context *ctx; - struct ibv_device_attr_ex device_attr; - struct ibv_port_attr port_attr; - int rc = 0; - uint8_t port; - char buf[256]; - - ctx = ibv_open_device(ib_dev); - if (!ctx) { - fprintf(stderr, "Failed to open device\n"); - rc = 1; - goto cleanup; - } - if (ibv_query_device_ex(ctx, NULL, &device_attr)) { - fprintf(stderr, "Failed to query device props\n"); - rc = 2; - goto cleanup; - } - if (ib_port && ib_port > device_attr.orig_attr.phys_port_cnt) { - fprintf(stderr, "Invalid port requested for device\n"); - /* rc = 3 is taken by failure to clean up */ - rc = 4; - goto cleanup; - } - - printf("hca_id:\t%s\n", ibv_get_device_name(ib_dev)); - printf("\ttransport:\t\t\t%s (%d)\n", - transport_str(ib_dev->transport_type), ib_dev->transport_type); - if (strlen(device_attr.orig_attr.fw_ver)) - printf("\tfw_ver:\t\t\t\t%s\n", device_attr.orig_attr.fw_ver); - printf("\tnode_guid:\t\t\t%s\n", guid_str(device_attr.orig_attr.node_guid, buf)); - printf("\tsys_image_guid:\t\t\t%s\n", guid_str(device_attr.orig_attr.sys_image_guid, buf)); - printf("\tvendor_id:\t\t\t0x%04x\n", device_attr.orig_attr.vendor_id); - printf("\tvendor_part_id:\t\t\t%d\n", device_attr.orig_attr.vendor_part_id); - printf("\thw_ver:\t\t\t\t0x%X\n", device_attr.orig_attr.hw_ver); - - if (ibv_read_sysfs_file(ib_dev->ibdev_path, "board_id", buf, sizeof buf) > 0) - printf("\tboard_id:\t\t\t%s\n", buf); - - printf("\tphys_port_cnt:\t\t\t%d\n", device_attr.orig_attr.phys_port_cnt); - - if (verbose) { - printf("\tmax_mr_size:\t\t\t0x%llx\n", - (unsigned long long) device_attr.orig_attr.max_mr_size); - printf("\tpage_size_cap:\t\t\t0x%llx\n", - (unsigned long long) device_attr.orig_attr.page_size_cap); - printf("\tmax_qp:\t\t\t\t%d\n", device_attr.orig_attr.max_qp); - printf("\tmax_qp_wr:\t\t\t%d\n", device_attr.orig_attr.max_qp_wr); - printf("\tdevice_cap_flags:\t\t0x%08x\n", device_attr.orig_attr.device_cap_flags); - print_device_cap_flags(device_attr.orig_attr.device_cap_flags); - printf("\tmax_sge:\t\t\t%d\n", device_attr.orig_attr.max_sge); - printf("\tmax_sge_rd:\t\t\t%d\n", device_attr.orig_attr.max_sge_rd); - printf("\tmax_cq:\t\t\t\t%d\n", device_attr.orig_attr.max_cq); - printf("\tmax_cqe:\t\t\t%d\n", device_attr.orig_attr.max_cqe); - printf("\tmax_mr:\t\t\t\t%d\n", device_attr.orig_attr.max_mr); - printf("\tmax_pd:\t\t\t\t%d\n", device_attr.orig_attr.max_pd); - printf("\tmax_qp_rd_atom:\t\t\t%d\n", device_attr.orig_attr.max_qp_rd_atom); - printf("\tmax_ee_rd_atom:\t\t\t%d\n", device_attr.orig_attr.max_ee_rd_atom); - printf("\tmax_res_rd_atom:\t\t%d\n", device_attr.orig_attr.max_res_rd_atom); - printf("\tmax_qp_init_rd_atom:\t\t%d\n", device_attr.orig_attr.max_qp_init_rd_atom); - printf("\tmax_ee_init_rd_atom:\t\t%d\n", device_attr.orig_attr.max_ee_init_rd_atom); - printf("\tatomic_cap:\t\t\t%s (%d)\n", - atomic_cap_str(device_attr.orig_attr.atomic_cap), device_attr.orig_attr.atomic_cap); - printf("\tmax_ee:\t\t\t\t%d\n", device_attr.orig_attr.max_ee); - printf("\tmax_rdd:\t\t\t%d\n", device_attr.orig_attr.max_rdd); - printf("\tmax_mw:\t\t\t\t%d\n", device_attr.orig_attr.max_mw); - printf("\tmax_raw_ipv6_qp:\t\t%d\n", device_attr.orig_attr.max_raw_ipv6_qp); - printf("\tmax_raw_ethy_qp:\t\t%d\n", device_attr.orig_attr.max_raw_ethy_qp); - printf("\tmax_mcast_grp:\t\t\t%d\n", device_attr.orig_attr.max_mcast_grp); - printf("\tmax_mcast_qp_attach:\t\t%d\n", device_attr.orig_attr.max_mcast_qp_attach); - printf("\tmax_total_mcast_qp_attach:\t%d\n", - device_attr.orig_attr.max_total_mcast_qp_attach); - printf("\tmax_ah:\t\t\t\t%d\n", device_attr.orig_attr.max_ah); - printf("\tmax_fmr:\t\t\t%d\n", device_attr.orig_attr.max_fmr); - if (device_attr.orig_attr.max_fmr) - printf("\tmax_map_per_fmr:\t\t%d\n", device_attr.orig_attr.max_map_per_fmr); - printf("\tmax_srq:\t\t\t%d\n", device_attr.orig_attr.max_srq); - if (device_attr.orig_attr.max_srq) { - printf("\tmax_srq_wr:\t\t\t%d\n", device_attr.orig_attr.max_srq_wr); - printf("\tmax_srq_sge:\t\t\t%d\n", device_attr.orig_attr.max_srq_sge); - } - printf("\tmax_pkeys:\t\t\t%d\n", device_attr.orig_attr.max_pkeys); - printf("\tlocal_ca_ack_delay:\t\t%d\n", device_attr.orig_attr.local_ca_ack_delay); - - print_odp_caps(&device_attr.odp_caps); - if (device_attr.completion_timestamp_mask) - printf("\tcompletion timestamp_mask:\t\t\t0x%016lx\n", - device_attr.completion_timestamp_mask); - else - printf("\tcompletion_timestamp_mask not supported\n"); - - if (device_attr.hca_core_clock) - printf("\thca_core_clock:\t\t\t%lukHZ\n", device_attr.hca_core_clock); - else - printf("\tcore clock not supported\n"); - - printf("\tdevice_cap_flags_ex:\t\t0x%" PRIX64 "\n", device_attr.device_cap_flags_ex); - print_device_cap_flags_ex(device_attr.device_cap_flags_ex); - print_tso_caps(&device_attr.tso_caps); - print_rss_caps(&device_attr.rss_caps); - printf("\tmax_wq_type_rq:\t\t\t%u\n", device_attr.max_wq_type_rq); - } - - for (port = 1; port <= device_attr.orig_attr.phys_port_cnt; ++port) { - /* if in the command line the user didn't ask for info about this port */ - if ((ib_port) && (port != ib_port)) - continue; - - rc = ibv_query_port(ctx, port, &port_attr); - if (rc) { - fprintf(stderr, "Failed to query port %u props\n", port); - goto cleanup; - } - printf("\t\tport:\t%d\n", port); - printf("\t\t\tstate:\t\t\t%s (%d)\n", - port_state_str(port_attr.state), port_attr.state); - printf("\t\t\tmax_mtu:\t\t%s (%d)\n", - mtu_str(port_attr.max_mtu), port_attr.max_mtu); - printf("\t\t\tactive_mtu:\t\t%s (%d)\n", - mtu_str(port_attr.active_mtu), port_attr.active_mtu); - printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid); - printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid); - printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc); - printf("\t\t\tlink_layer:\t\t%s\n", - link_layer_str(port_attr.link_layer)); - - if (verbose) { - printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz); - printf("\t\t\tport_cap_flags:\t\t0x%08x\n", port_attr.port_cap_flags); - printf("\t\t\tmax_vl_num:\t\t%s (%d)\n", - vl_str(port_attr.max_vl_num), port_attr.max_vl_num); - printf("\t\t\tbad_pkey_cntr:\t\t0x%x\n", port_attr.bad_pkey_cntr); - printf("\t\t\tqkey_viol_cntr:\t\t0x%x\n", port_attr.qkey_viol_cntr); - printf("\t\t\tsm_sl:\t\t\t%d\n", port_attr.sm_sl); - printf("\t\t\tpkey_tbl_len:\t\t%d\n", port_attr.pkey_tbl_len); - printf("\t\t\tgid_tbl_len:\t\t%d\n", port_attr.gid_tbl_len); - printf("\t\t\tsubnet_timeout:\t\t%d\n", port_attr.subnet_timeout); - printf("\t\t\tinit_type_reply:\t%d\n", port_attr.init_type_reply); - printf("\t\t\tactive_width:\t\t%sX (%d)\n", - width_str(port_attr.active_width), port_attr.active_width); - printf("\t\t\tactive_speed:\t\t%s (%d)\n", - speed_str(port_attr.active_speed), port_attr.active_speed); - if (ib_dev->transport_type == IBV_TRANSPORT_IB) - printf("\t\t\tphys_state:\t\t%s (%d)\n", - port_phy_state_str(port_attr.phys_state), port_attr.phys_state); - - if (print_all_port_gids(ctx, port, port_attr.gid_tbl_len)) - goto cleanup; - } - printf("\n"); - } -cleanup: - if (ctx) - if (ibv_close_device(ctx)) { - fprintf(stderr, "Failed to close device"); - rc = 3; - } - return rc; -} - -static void usage(const char *argv0) -{ - printf("Usage: %s print the ca attributes\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default all ports)\n"); - printf(" -l, --list print only the IB devices names\n"); - printf(" -v, --verbose print all the attributes of the IB device(s)\n"); -} - -int main(int argc, char *argv[]) -{ - char *ib_devname = NULL; - int ret = 0; - struct ibv_device **dev_list, **orig_dev_list; - int num_of_hcas; - int ib_port = 0; - - /* parse command line options */ - while (1) { - int c; - static struct option long_options[] = { - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "list", .has_arg = 0, .val = 'l' }, - { .name = "verbose", .has_arg = 0, .val = 'v' }, - { 0, 0, 0, 0} - }; - - c = getopt_long(argc, argv, "d:i:lv", long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'd': - ib_devname = strdup(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port <= 0) { - usage(argv[0]); - return 1; - } - break; - - case 'v': - verbose = 1; - break; - - case 'l': - dev_list = orig_dev_list = ibv_get_device_list(&num_of_hcas); - if (!dev_list) { - perror("Failed to get IB devices list"); - return -1; - } - - printf("%d HCA%s found:\n", num_of_hcas, - num_of_hcas != 1 ? "s" : ""); - - while (*dev_list) { - printf("\t%s\n", ibv_get_device_name(*dev_list)); - ++dev_list; - } - - printf("\n"); - - ibv_free_device_list(orig_dev_list); - - return 0; - - default: - usage(argv[0]); - return -1; - } - } - - dev_list = orig_dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return -1; - } - - if (ib_devname) { - while (*dev_list) { - if (!strcmp(ibv_get_device_name(*dev_list), ib_devname)) - break; - ++dev_list; - } - - if (!*dev_list) { - fprintf(stderr, "IB device '%s' wasn't found\n", ib_devname); - return -1; - } - - ret |= print_hca_cap(*dev_list, ib_port); - } else { - if (!*dev_list) { - fprintf(stderr, "No IB devices found\n"); - return -1; - } - - while (*dev_list) { - ret |= print_hca_cap(*dev_list, ib_port); - ++dev_list; - } - } - - if (ib_devname) - free(ib_devname); - - ibv_free_device_list(orig_dev_list); - - return ret; -} diff --git a/examples/pingpong.c b/examples/pingpong.c deleted file mode 100644 index 2fe4a04..0000000 --- a/examples/pingpong.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2006 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "pingpong.h" -#include <arpa/inet.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> - -enum ibv_mtu pp_mtu_to_enum(int mtu) -{ - switch (mtu) { - case 256: return IBV_MTU_256; - case 512: return IBV_MTU_512; - case 1024: return IBV_MTU_1024; - case 2048: return IBV_MTU_2048; - case 4096: return IBV_MTU_4096; - default: return -1; - } -} - -int pp_get_port_info(struct ibv_context *context, int port, - struct ibv_port_attr *attr) -{ - return ibv_query_port(context, port, attr); -} - -void wire_gid_to_gid(const char *wgid, union ibv_gid *gid) -{ - char tmp[9]; - uint32_t v32; - int i; - - for (tmp[8] = 0, i = 0; i < 4; ++i) { - memcpy(tmp, wgid + i * 8, 8); - sscanf(tmp, "%x", &v32); - *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); - } -} - -void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]) -{ - int i; - - for (i = 0; i < 4; ++i) - sprintf(&wgid[i * 8], "%08x", - htonl(*(uint32_t *)(gid->raw + i * 4))); -} diff --git a/examples/pingpong.h b/examples/pingpong.h deleted file mode 100644 index 8dc5dd0..0000000 --- a/examples/pingpong.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2006 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef IBV_PINGPONG_H -#define IBV_PINGPONG_H - -#include <infiniband/verbs.h> - -enum ibv_mtu pp_mtu_to_enum(int mtu); -int pp_get_port_info(struct ibv_context *context, int port, - struct ibv_port_attr *attr); -void wire_gid_to_gid(const char *wgid, union ibv_gid *gid); -void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]); - -#endif /* IBV_PINGPONG_H */ diff --git a/examples/rc_pingpong.c b/examples/rc_pingpong.c deleted file mode 100644 index 8d58357..0000000 --- a/examples/rc_pingpong.c +++ /dev/null @@ -1,1049 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <netdb.h> -#include <malloc.h> -#include <getopt.h> -#include <arpa/inet.h> -#include <time.h> - -#include "pingpong.h" - -#ifndef max -#define max(x, y) (((x) > (y)) ? (x) : (y)) -#endif - -#ifndef min -#define min(x, y) (((x) < (y)) ? (x) : (y)) -#endif - -enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, -}; - -static int page_size; -static int use_odp; -static int use_ts; - -struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - union { - struct ibv_cq *cq; - struct ibv_cq_ex *cq_ex; - } cq_s; - struct ibv_qp *qp; - void *buf; - int size; - int send_flags; - int rx_depth; - int pending; - struct ibv_port_attr portinfo; - uint64_t completion_timestamp_mask; -}; - -struct ibv_cq *pp_cq(struct pingpong_context *ctx) -{ - return use_ts ? ibv_cq_ex_to_cq(ctx->cq_s.cq_ex) : - ctx->cq_s.cq; -} - -struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; -}; - -static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - enum ibv_mtu mtu, int sl, - struct pingpong_dest *dest, int sgid_idx) -{ - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR, - .path_mtu = mtu, - .dest_qp_num = dest->qpn, - .rq_psn = dest->psn, - .max_dest_rd_atomic = 1, - .min_rnr_timer = 12, - .ah_attr = { - .is_global = 0, - .dlid = dest->lid, - .sl = sl, - .src_path_bits = 0, - .port_num = port - } - }; - - if (dest->gid.global.interface_id) { - attr.ah_attr.is_global = 1; - attr.ah_attr.grh.hop_limit = 1; - attr.ah_attr.grh.dgid = dest->gid; - attr.ah_attr.grh.sgid_index = sgid_idx; - } - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_AV | - IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | - IBV_QP_RQ_PSN | - IBV_QP_MAX_DEST_RD_ATOMIC | - IBV_QP_MIN_RNR_TIMER)) { - fprintf(stderr, "Failed to modify QP to RTR\n"); - return 1; - } - - attr.qp_state = IBV_QPS_RTS; - attr.timeout = 14; - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.sq_psn = my_psn; - attr.max_rd_atomic = 1; - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_TIMEOUT | - IBV_QP_RETRY_CNT | - IBV_QP_RNR_RETRY | - IBV_QP_SQ_PSN | - IBV_QP_MAX_QP_RD_ATOMIC)) { - fprintf(stderr, "Failed to modify QP to RTS\n"); - return 1; - } - - return 0; -} - -static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, - const struct pingpong_dest *my_dest) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(servername, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); - return NULL; - } - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(sockfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - goto out; - } - - if (read(sockfd, msg, sizeof msg) != sizeof msg) { - perror("client read"); - fprintf(stderr, "Couldn't read remote address\n"); - goto out; - } - - write(sockfd, "done", sizeof "done"); - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - -out: - close(sockfd); - return rem_dest; -} - -static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, - int ib_port, enum ibv_mtu mtu, - int port, int sl, - const struct pingpong_dest *my_dest, - int sgid_idx) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_flags = AI_PASSIVE, - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1, connfd; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(NULL, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - n = 1; - - setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); - - if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't listen to port %d\n", port); - return NULL; - } - - listen(sockfd, 1); - connfd = accept(sockfd, NULL, 0); - close(sockfd); - if (connfd < 0) { - fprintf(stderr, "accept() failed\n"); - return NULL; - } - - n = read(connfd, msg, sizeof msg); - if (n != sizeof msg) { - perror("server read"); - fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); - goto out; - } - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, - sgid_idx)) { - fprintf(stderr, "Couldn't connect to remote QP\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(connfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - read(connfd, msg, sizeof msg); - -out: - close(connfd); - return rem_dest; -} - -static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, - int rx_depth, int port, - int use_event) -{ - struct pingpong_context *ctx; - int access_flags = IBV_ACCESS_LOCAL_WRITE; - - ctx = calloc(1, sizeof *ctx); - if (!ctx) - return NULL; - - ctx->size = size; - ctx->send_flags = IBV_SEND_SIGNALED; - ctx->rx_depth = rx_depth; - - ctx->buf = memalign(page_size, size); - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } - - /* FIXME memset(ctx->buf, 0, size); */ - memset(ctx->buf, 0x7b, size); - - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - if (use_odp || use_ts) { - const uint32_t rc_caps_mask = IBV_ODP_SUPPORT_SEND | - IBV_ODP_SUPPORT_RECV; - struct ibv_device_attr_ex attrx; - - if (ibv_query_device_ex(ctx->context, NULL, &attrx)) { - fprintf(stderr, "Couldn't query device for its features\n"); - goto clean_comp_channel; - } - - if (use_odp) { - if (!(attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) || - (attrx.odp_caps.per_transport_caps.rc_odp_caps & rc_caps_mask) != rc_caps_mask) { - fprintf(stderr, "The device isn't ODP capable or does not support RC send and receive with ODP\n"); - goto clean_comp_channel; - } - access_flags |= IBV_ACCESS_ON_DEMAND; - } - - if (use_ts) { - if (!attrx.completion_timestamp_mask) { - fprintf(stderr, "The device isn't completion timestamp capable\n"); - goto clean_comp_channel; - } - ctx->completion_timestamp_mask = attrx.completion_timestamp_mask; - } - } - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, access_flags); - - if (!ctx->mr) { - fprintf(stderr, "Couldn't register MR\n"); - goto clean_pd; - } - - if (use_ts) { - struct ibv_cq_init_attr_ex attr_ex = { - .cqe = rx_depth + 1, - .cq_context = NULL, - .channel = ctx->channel, - .comp_vector = 0, - .wc_flags = IBV_WC_EX_WITH_COMPLETION_TIMESTAMP - }; - - ctx->cq_s.cq_ex = ibv_create_cq_ex(ctx->context, &attr_ex); - } else { - ctx->cq_s.cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, - ctx->channel, 0); - } - - if (!pp_cq(ctx)) { - fprintf(stderr, "Couldn't create CQ\n"); - goto clean_mr; - } - - { - struct ibv_qp_attr attr; - struct ibv_qp_init_attr init_attr = { - .send_cq = pp_cq(ctx), - .recv_cq = pp_cq(ctx), - .cap = { - .max_send_wr = 1, - .max_recv_wr = rx_depth, - .max_send_sge = 1, - .max_recv_sge = 1 - }, - .qp_type = IBV_QPT_RC - }; - - ctx->qp = ibv_create_qp(ctx->pd, &init_attr); - if (!ctx->qp) { - fprintf(stderr, "Couldn't create QP\n"); - goto clean_cq; - } - - ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr); - if (init_attr.cap.max_inline_data >= size) { - ctx->send_flags |= IBV_SEND_INLINE; - } - } - - { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qp_access_flags = 0 - }; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_ACCESS_FLAGS)) { - fprintf(stderr, "Failed to modify QP to INIT\n"); - goto clean_qp; - } - } - - return ctx; - -clean_qp: - ibv_destroy_qp(ctx->qp); - -clean_cq: - ibv_destroy_cq(pp_cq(ctx)); - -clean_mr: - ibv_dereg_mr(ctx->mr); - -clean_pd: - ibv_dealloc_pd(ctx->pd); - -clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); - -clean_device: - ibv_close_device(ctx->context); - -clean_buffer: - free(ctx->buf); - -clean_ctx: - free(ctx); - - return NULL; -} - -int pp_close_ctx(struct pingpong_context *ctx) -{ - if (ibv_destroy_qp(ctx->qp)) { - fprintf(stderr, "Couldn't destroy QP\n"); - return 1; - } - - if (ibv_destroy_cq(pp_cq(ctx))) { - fprintf(stderr, "Couldn't destroy CQ\n"); - return 1; - } - - if (ibv_dereg_mr(ctx->mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - return 1; - } - - if (ibv_dealloc_pd(ctx->pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - return 1; - } - - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - fprintf(stderr, "Couldn't destroy completion channel\n"); - return 1; - } - } - - if (ibv_close_device(ctx->context)) { - fprintf(stderr, "Couldn't release context\n"); - return 1; - } - - free(ctx->buf); - free(ctx); - - return 0; -} - -static int pp_post_recv(struct pingpong_context *ctx, int n) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; - - for (i = 0; i < n; ++i) - if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) - break; - - return i; -} - -static int pp_post_send(struct pingpong_context *ctx) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_send_wr wr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .opcode = IBV_WR_SEND, - .send_flags = ctx->send_flags, - }; - struct ibv_send_wr *bad_wr; - - return ibv_post_send(ctx->qp, &wr, &bad_wr); -} - -struct ts_params { - unsigned int comp_recv_max_time_delta; - unsigned int comp_recv_min_time_delta; - uint64_t comp_recv_total_time_delta; - uint64_t comp_recv_prev_time; - int last_comp_with_ts; - unsigned int comp_with_time_iters; -}; - -static inline int parse_single_wc(struct pingpong_context *ctx, int *scnt, - int *rcnt, int *routs, int iters, - uint64_t wr_id, enum ibv_wc_status status, - uint64_t completion_timestamp, - struct ts_params *ts) -{ - if (status != IBV_WC_SUCCESS) { - fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(status), - status, (int)wr_id); - return 1; - } - - switch ((int)wr_id) { - case PINGPONG_SEND_WRID: - ++(*scnt); - break; - - case PINGPONG_RECV_WRID: - if (--(*routs) <= 1) { - *routs += pp_post_recv(ctx, ctx->rx_depth - *routs); - if (*routs < ctx->rx_depth) { - fprintf(stderr, - "Couldn't post receive (%d)\n", - *routs); - return 1; - } - } - - ++(*rcnt); - if (use_ts) { - if (ts->last_comp_with_ts) { - uint64_t delta; - - /* checking whether the clock was wrapped around */ - if (completion_timestamp >= ts->comp_recv_prev_time) - delta = completion_timestamp - ts->comp_recv_prev_time; - else - delta = ctx->completion_timestamp_mask - ts->comp_recv_prev_time + - completion_timestamp + 1; - - ts->comp_recv_max_time_delta = max(ts->comp_recv_max_time_delta, delta); - ts->comp_recv_min_time_delta = min(ts->comp_recv_min_time_delta, delta); - ts->comp_recv_total_time_delta += delta; - ts->comp_with_time_iters++; - } - - ts->comp_recv_prev_time = completion_timestamp; - ts->last_comp_with_ts = 1; - } else { - ts->last_comp_with_ts = 0; - } - - break; - - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int)wr_id); - return 1; - } - - ctx->pending &= ~(int)wr_id; - if (*scnt < iters && !ctx->pending) { - if (pp_post_send(ctx)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending = PINGPONG_RECV_WRID | - PINGPONG_SEND_WRID; - } - - return 0; -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s <host> connect to server at <host>\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); - printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); - printf(" -m, --mtu=<size> path MTU (default 1024)\n"); - printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); - printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); - printf(" -l, --sl=<sl> service level value\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx=<gid index> local port gid index\n"); - printf(" -o, --odp use on demand paging\n"); - printf(" -t, --ts get CQE with timestamp\n"); -} - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct pingpong_context *ctx; - struct pingpong_dest my_dest; - struct pingpong_dest *rem_dest; - struct timeval start, end; - char *ib_devname = NULL; - char *servername = NULL; - unsigned int port = 18515; - int ib_port = 1; - unsigned int size = 4096; - enum ibv_mtu mtu = IBV_MTU_1024; - unsigned int rx_depth = 500; - unsigned int iters = 1000; - int use_event = 0; - int routs; - int rcnt, scnt; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[33]; - struct ts_params ts; - - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "mtu", .has_arg = 1, .val = 'm' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { .name = "odp", .has_arg = 0, .val = 'o' }, - { .name = "ts", .has_arg = 0, .val = 't' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:ot", - long_options, NULL); - - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtoul(optarg, NULL, 0); - if (port > 65535) { - usage(argv[0]); - return 1; - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 1) { - usage(argv[0]); - return 1; - } - break; - - case 's': - size = strtoul(optarg, NULL, 0); - break; - - case 'm': - mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); - if (mtu < 0) { - usage(argv[0]); - return 1; - } - break; - - case 'r': - rx_depth = strtoul(optarg, NULL, 0); - break; - - case 'n': - iters = strtoul(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - case 'o': - use_odp = 1; - break; - case 't': - use_ts = 1; - break; - - default: - usage(argv[0]); - return 1; - } - } - - if (optind == argc - 1) - servername = strdupa(argv[optind]); - else if (optind < argc) { - usage(argv[0]); - return 1; - } - - if (use_ts) { - ts.comp_recv_max_time_delta = 0; - ts.comp_recv_min_time_delta = 0xffffffff; - ts.comp_recv_total_time_delta = 0; - ts.comp_recv_prev_time = 0; - ts.last_comp_with_ts = 0; - ts.comp_with_time_iters = 0; - } - - page_size = sysconf(_SC_PAGESIZE); - - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - - if (!ib_devname) { - ib_dev = *dev_list; - if (!ib_dev) { - fprintf(stderr, "No IB devices found\n"); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - fprintf(stderr, "IB device %s not found\n", ib_devname); - return 1; - } - } - - ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); - if (!ctx) - return 1; - - routs = pp_post_recv(ctx, ctx->rx_depth); - if (routs < ctx->rx_depth) { - fprintf(stderr, "Couldn't post receive (%d)\n", routs); - return 1; - } - - if (use_event) - if (ibv_req_notify_cq(pp_cq(ctx), 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - fprintf(stderr, "Couldn't get port info\n"); - return 1; - } - - my_dest.lid = ctx->portinfo.lid; - if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && - !my_dest.lid) { - fprintf(stderr, "Couldn't get local LID\n"); - return 1; - } - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { - fprintf(stderr, "can't read sgid of index %d\n", gidx); - return 1; - } - } else - memset(&my_dest.gid, 0, sizeof my_dest.gid); - - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - my_dest.lid, my_dest.qpn, my_dest.psn, gid); - - - if (servername) - rem_dest = pp_client_exch_dest(servername, port, &my_dest); - else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, - &my_dest, gidx); - - if (!rem_dest) - return 1; - - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); - - if (servername) - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, - gidx)) - return 1; - - ctx->pending = PINGPONG_RECV_WRID; - - if (servername) { - if (pp_post_send(ctx)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending |= PINGPONG_SEND_WRID; - } - - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } - - rcnt = scnt = 0; - while (rcnt < iters || scnt < iters) { - int ret; - - if (use_event) { - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\n"); - return 1; - } - - ++num_cq_events; - - if (ev_cq != pp_cq(ctx)) { - fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(pp_cq(ctx), 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - } - - if (use_ts) { - struct ibv_poll_cq_attr attr = {}; - - do { - ret = ibv_start_poll(ctx->cq_s.cq_ex, &attr); - } while (!use_event && ret == ENOENT); - - if (ret) { - fprintf(stderr, "poll CQ failed %d\n", ret); - return ret; - } - ret = parse_single_wc(ctx, &scnt, &rcnt, &routs, - iters, - ctx->cq_s.cq_ex->wr_id, - ctx->cq_s.cq_ex->status, - ibv_wc_read_completion_ts(ctx->cq_s.cq_ex), - &ts); - if (ret) { - ibv_end_poll(ctx->cq_s.cq_ex); - return ret; - } - ret = ibv_next_poll(ctx->cq_s.cq_ex); - if (!ret) - ret = parse_single_wc(ctx, &scnt, &rcnt, &routs, - iters, - ctx->cq_s.cq_ex->wr_id, - ctx->cq_s.cq_ex->status, - ibv_wc_read_completion_ts(ctx->cq_s.cq_ex), - &ts); - ibv_end_poll(ctx->cq_s.cq_ex); - if (ret && ret != ENOENT) { - fprintf(stderr, "poll CQ failed %d\n", ret); - return ret; - } - } else { - int ne, i; - struct ibv_wc wc[2]; - - do { - ne = ibv_poll_cq(pp_cq(ctx), 2, wc); - if (ne < 0) { - fprintf(stderr, "poll CQ failed %d\n", ne); - return 1; - } - } while (!use_event && ne < 1); - - for (i = 0; i < ne; ++i) { - ret = parse_single_wc(ctx, &scnt, &rcnt, &routs, - iters, - wc[i].wr_id, - wc[i].status, - 0, &ts); - if (ret) { - fprintf(stderr, "parse WC failed %d\n", ne); - return 1; - } - } - } - } - - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - return 1; - } - - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - long long bytes = (long long) size * iters * 2; - - printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", - bytes, usec / 1000000., bytes * 8. / usec); - printf("%d iters in %.2f seconds = %.2f usec/iter\n", - iters, usec / 1000000., usec / iters); - - if (use_ts && ts.comp_with_time_iters) { - printf("Max receive completion clock cycles = %u\n", - ts.comp_recv_max_time_delta); - printf("Min receive completion clock cycles = %u\n", - ts.comp_recv_min_time_delta); - printf("Average receive completion clock cycles = %f\n", - (double)ts.comp_recv_total_time_delta / ts.comp_with_time_iters); - } - } - - ibv_ack_cq_events(pp_cq(ctx), num_cq_events); - - if (pp_close_ctx(ctx)) - return 1; - - ibv_free_device_list(dev_list); - free(rem_dest); - - return 0; -} diff --git a/examples/srq_pingpong.c b/examples/srq_pingpong.c deleted file mode 100644 index f61acb0..0000000 --- a/examples/srq_pingpong.c +++ /dev/null @@ -1,981 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <netdb.h> -#include <malloc.h> -#include <getopt.h> -#include <arpa/inet.h> -#include <time.h> - -#include "pingpong.h" - -enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, - - MAX_QP = 256, -}; - -static int page_size; - -struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *cq; - struct ibv_srq *srq; - struct ibv_qp *qp[MAX_QP]; - void *buf; - int size; - int send_flags; - int num_qp; - int rx_depth; - int pending[MAX_QP]; - struct ibv_port_attr portinfo; -}; - -struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; -}; - -static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu, - int sl, const struct pingpong_dest *my_dest, - const struct pingpong_dest *dest, int sgid_idx) -{ - int i; - - for (i = 0; i < ctx->num_qp; ++i) { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR, - .path_mtu = mtu, - .dest_qp_num = dest[i].qpn, - .rq_psn = dest[i].psn, - .max_dest_rd_atomic = 1, - .min_rnr_timer = 12, - .ah_attr = { - .is_global = 0, - .dlid = dest[i].lid, - .sl = sl, - .src_path_bits = 0, - .port_num = port - } - }; - - if (dest->gid.global.interface_id) { - attr.ah_attr.is_global = 1; - attr.ah_attr.grh.hop_limit = 1; - attr.ah_attr.grh.dgid = dest->gid; - attr.ah_attr.grh.sgid_index = sgid_idx; - } - if (ibv_modify_qp(ctx->qp[i], &attr, - IBV_QP_STATE | - IBV_QP_AV | - IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | - IBV_QP_RQ_PSN | - IBV_QP_MAX_DEST_RD_ATOMIC | - IBV_QP_MIN_RNR_TIMER)) { - fprintf(stderr, "Failed to modify QP[%d] to RTR\n", i); - return 1; - } - - attr.qp_state = IBV_QPS_RTS; - attr.timeout = 14; - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.sq_psn = my_dest[i].psn; - attr.max_rd_atomic = 1; - if (ibv_modify_qp(ctx->qp[i], &attr, - IBV_QP_STATE | - IBV_QP_TIMEOUT | - IBV_QP_RETRY_CNT | - IBV_QP_RNR_RETRY | - IBV_QP_SQ_PSN | - IBV_QP_MAX_QP_RD_ATOMIC)) { - fprintf(stderr, "Failed to modify QP[%d] to RTS\n", i); - return 1; - } - } - - return 0; -} - -static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, - const struct pingpong_dest *my_dest) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int r; - int i; - int sockfd = -1; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(servername, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); - return NULL; - } - - for (i = 0; i < MAX_QP; ++i) { - gid_to_wire_gid(&my_dest[i].gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, - my_dest[i].qpn, my_dest[i].psn, gid); - if (write(sockfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - goto out; - } - } - - rem_dest = malloc(MAX_QP * sizeof *rem_dest); - if (!rem_dest) - goto out; - - for (i = 0; i < MAX_QP; ++i) { - n = 0; - while (n < sizeof msg) { - r = read(sockfd, msg + n, sizeof msg - n); - if (r < 0) { - perror("client read"); - fprintf(stderr, "%d/%d: Couldn't read remote address [%d]\n", - n, (int) sizeof msg, i); - goto out; - } - n += r; - } - - sscanf(msg, "%x:%x:%x:%s", &rem_dest[i].lid, &rem_dest[i].qpn, - &rem_dest[i].psn, gid); - wire_gid_to_gid(gid, &rem_dest[i].gid); - } - - write(sockfd, "done", sizeof "done"); - -out: - close(sockfd); - return rem_dest; -} - -static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, - int ib_port, enum ibv_mtu mtu, - int port, int sl, - const struct pingpong_dest *my_dest, - int sgid_idx) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_flags = AI_PASSIVE, - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int r; - int i; - int sockfd = -1, connfd; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(NULL, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - n = 1; - - setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); - - if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't listen to port %d\n", port); - return NULL; - } - - listen(sockfd, 1); - connfd = accept(sockfd, NULL, 0); - close(sockfd); - if (connfd < 0) { - fprintf(stderr, "accept() failed\n"); - return NULL; - } - - rem_dest = malloc(MAX_QP * sizeof *rem_dest); - if (!rem_dest) - goto out; - - for (i = 0; i < MAX_QP; ++i) { - n = 0; - while (n < sizeof msg) { - r = read(connfd, msg + n, sizeof msg - n); - if (r < 0) { - perror("server read"); - fprintf(stderr, "%d/%d: Couldn't read remote address [%d]\n", - n, (int) sizeof msg, i); - goto out; - } - n += r; - } - - sscanf(msg, "%x:%x:%x:%s", &rem_dest[i].lid, &rem_dest[i].qpn, - &rem_dest[i].psn, gid); - wire_gid_to_gid(gid, &rem_dest[i].gid); - } - - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, - sgid_idx)) { - fprintf(stderr, "Couldn't connect to remote QP\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - for (i = 0; i < MAX_QP; ++i) { - gid_to_wire_gid(&my_dest[i].gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, - my_dest[i].qpn, my_dest[i].psn, gid); - if (write(connfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - } - - read(connfd, msg, sizeof msg); - -out: - close(connfd); - return rem_dest; -} - -static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, - int num_qp, int rx_depth, int port, - int use_event) -{ - struct pingpong_context *ctx; - int i; - - ctx = calloc(1, sizeof *ctx); - if (!ctx) - return NULL; - - ctx->size = size; - ctx->send_flags = IBV_SEND_SIGNALED; - ctx->num_qp = num_qp; - ctx->rx_depth = rx_depth; - - ctx->buf = memalign(page_size, size); - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } - - memset(ctx->buf, 0, size); - - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); - if (!ctx->mr) { - fprintf(stderr, "Couldn't register MR\n"); - goto clean_pd; - } - - ctx->cq = ibv_create_cq(ctx->context, rx_depth + num_qp, NULL, - ctx->channel, 0); - if (!ctx->cq) { - fprintf(stderr, "Couldn't create CQ\n"); - goto clean_mr; - } - - { - struct ibv_srq_init_attr attr = { - .attr = { - .max_wr = rx_depth, - .max_sge = 1 - } - }; - - ctx->srq = ibv_create_srq(ctx->pd, &attr); - if (!ctx->srq) { - fprintf(stderr, "Couldn't create SRQ\n"); - goto clean_cq; - } - } - - for (i = 0; i < num_qp; ++i) { - struct ibv_qp_attr attr; - struct ibv_qp_init_attr init_attr = { - .send_cq = ctx->cq, - .recv_cq = ctx->cq, - .srq = ctx->srq, - .cap = { - .max_send_wr = 1, - .max_send_sge = 1, - }, - .qp_type = IBV_QPT_RC - }; - - ctx->qp[i] = ibv_create_qp(ctx->pd, &init_attr); - if (!ctx->qp[i]) { - fprintf(stderr, "Couldn't create QP[%d]\n", i); - goto clean_qps; - } - ibv_query_qp(ctx->qp[i], &attr, IBV_QP_CAP, &init_attr); - if (init_attr.cap.max_inline_data >= size) { - ctx->send_flags |= IBV_SEND_INLINE; - } - } - - for (i = 0; i < num_qp; ++i) { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qp_access_flags = 0 - }; - - if (ibv_modify_qp(ctx->qp[i], &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_ACCESS_FLAGS)) { - fprintf(stderr, "Failed to modify QP[%d] to INIT\n", i); - goto clean_qps_full; - } - } - - return ctx; - -clean_qps_full: - i = num_qp; - -clean_qps: - for (--i; i >= 0; --i) - ibv_destroy_qp(ctx->qp[i]); - - ibv_destroy_srq(ctx->srq); - -clean_cq: - ibv_destroy_cq(ctx->cq); - -clean_mr: - ibv_dereg_mr(ctx->mr); - -clean_pd: - ibv_dealloc_pd(ctx->pd); - -clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); - -clean_device: - ibv_close_device(ctx->context); - -clean_buffer: - free(ctx->buf); - -clean_ctx: - free(ctx); - - return NULL; -} - -int pp_close_ctx(struct pingpong_context *ctx, int num_qp) -{ - int i; - - for (i = 0; i < num_qp; ++i) { - if (ibv_destroy_qp(ctx->qp[i])) { - fprintf(stderr, "Couldn't destroy QP[%d]\n", i); - return 1; - } - } - - if (ibv_destroy_srq(ctx->srq)) { - fprintf(stderr, "Couldn't destroy SRQ\n"); - return 1; - } - - if (ibv_destroy_cq(ctx->cq)) { - fprintf(stderr, "Couldn't destroy CQ\n"); - return 1; - } - - if (ibv_dereg_mr(ctx->mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - return 1; - } - - if (ibv_dealloc_pd(ctx->pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - return 1; - } - - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - fprintf(stderr, "Couldn't destroy completion channel\n"); - return 1; - } - } - - if (ibv_close_device(ctx->context)) { - fprintf(stderr, "Couldn't release context\n"); - return 1; - } - - free(ctx->buf); - free(ctx); - - return 0; -} - -static int pp_post_recv(struct pingpong_context *ctx, int n) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; - - for (i = 0; i < n; ++i) - if (ibv_post_srq_recv(ctx->srq, &wr, &bad_wr)) - break; - - return i; -} - -static int pp_post_send(struct pingpong_context *ctx, int qp_index) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_send_wr wr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .opcode = IBV_WR_SEND, - .send_flags = ctx->send_flags, - }; - struct ibv_send_wr *bad_wr; - - return ibv_post_send(ctx->qp[qp_index], &wr, &bad_wr); -} - -static int find_qp(int qpn, struct pingpong_context *ctx, int num_qp) -{ - int i; - - for (i = 0; i < num_qp; ++i) - if (ctx->qp[i]->qp_num == qpn) - return i; - - return -1; -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s <host> connect to server at <host>\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); - printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); - printf(" -m, --mtu=<size> path MTU (default 1024)\n"); - printf(" -q, --num-qp=<num> number of QPs to use (default 16)\n"); - printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); - printf(" -n, --iters=<iters> number of exchanges per QP(default 1000)\n"); - printf(" -l, --sl=<sl> service level value\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx=<gid index> local port gid index\n"); -} - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct ibv_wc *wc; - struct pingpong_context *ctx; - struct pingpong_dest my_dest[MAX_QP]; - struct pingpong_dest *rem_dest; - struct timeval start, end; - char *ib_devname = NULL; - char *servername = NULL; - unsigned int port = 18515; - int ib_port = 1; - unsigned int size = 4096; - enum ibv_mtu mtu = IBV_MTU_1024; - unsigned int num_qp = 16; - unsigned int rx_depth = 500; - unsigned int iters = 1000; - int use_event = 0; - int routs; - int rcnt, scnt; - int num_wc; - int i; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[33]; - - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "mtu", .has_arg = 1, .val = 'm' }, - { .name = "num-qp", .has_arg = 1, .val = 'q' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", - long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtoul(optarg, NULL, 0); - if (port > 65535) { - usage(argv[0]); - return 1; - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 1) { - usage(argv[0]); - return 1; - } - break; - - case 's': - size = strtoul(optarg, NULL, 0); - if (size < 1) { - usage(argv[0]); - return 1; - } - break; - - case 'm': - mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); - if (mtu < 0) { - usage(argv[0]); - return 1; - } - break; - - case 'q': - num_qp = strtoul(optarg, NULL, 0); - break; - - case 'r': - rx_depth = strtoul(optarg, NULL, 0); - break; - - case 'n': - iters = strtoul(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - default: - usage(argv[0]); - return 1; - } - } - - if (optind == argc - 1) - servername = strdupa(argv[optind]); - else if (optind < argc) { - usage(argv[0]); - return 1; - } - - if (num_qp > rx_depth) { - fprintf(stderr, "rx_depth %d is too small for %d QPs -- " - "must have at least one receive per QP.\n", - rx_depth, num_qp); - return 1; - } - - num_wc = num_qp + rx_depth; - wc = alloca(num_wc * sizeof *wc); - - page_size = sysconf(_SC_PAGESIZE); - - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - - if (!ib_devname) { - ib_dev = *dev_list; - if (!ib_dev) { - fprintf(stderr, "No IB devices found\n"); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - fprintf(stderr, "IB device %s not found\n", ib_devname); - return 1; - } - } - - ctx = pp_init_ctx(ib_dev, size, num_qp, rx_depth, ib_port, use_event); - if (!ctx) - return 1; - - routs = pp_post_recv(ctx, ctx->rx_depth); - if (routs < ctx->rx_depth) { - fprintf(stderr, "Couldn't post receive (%d)\n", routs); - return 1; - } - - if (use_event) - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - - memset(my_dest, 0, sizeof my_dest); - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - fprintf(stderr, "Couldn't get port info\n"); - return 1; - } - for (i = 0; i < num_qp; ++i) { - my_dest[i].qpn = ctx->qp[i]->qp_num; - my_dest[i].psn = lrand48() & 0xffffff; - my_dest[i].lid = ctx->portinfo.lid; - if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET - && !my_dest[i].lid) { - fprintf(stderr, "Couldn't get local LID\n"); - return 1; - } - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, - &my_dest[i].gid)) { - fprintf(stderr, "Could not get local gid for " - "gid index %d\n", gidx); - return 1; - } - } else - memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid); - - inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid); - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, " - "GID %s\n", my_dest[i].lid, my_dest[i].qpn, - my_dest[i].psn, gid); - } - - if (servername) - rem_dest = pp_client_exch_dest(servername, port, my_dest); - else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, - my_dest, gidx); - - if (!rem_dest) - return 1; - - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - - for (i = 0; i < num_qp; ++i) { - inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid); - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, " - "GID %s\n", rem_dest[i].lid, rem_dest[i].qpn, - rem_dest[i].psn, gid); - } - - if (servername) - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, - gidx)) - return 1; - - if (servername) - for (i = 0; i < num_qp; ++i) { - if (pp_post_send(ctx, i)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending[i] = PINGPONG_SEND_WRID | PINGPONG_RECV_WRID; - } - else - for (i = 0; i < num_qp; ++i) - ctx->pending[i] = PINGPONG_RECV_WRID; - - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } - - rcnt = scnt = 0; - while (rcnt < iters || scnt < iters) { - if (use_event) { - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\n"); - return 1; - } - - ++num_cq_events; - - if (ev_cq != ctx->cq) { - fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - } - - { - int ne, qp_ind; - - do { - ne = ibv_poll_cq(ctx->cq, num_wc, wc); - if (ne < 0) { - fprintf(stderr, "poll CQ failed %d\n", ne); - return 1; - } - } while (!use_event && ne < 1); - - for (i = 0; i < ne; ++i) { - if (wc[i].status != IBV_WC_SUCCESS) { - fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); - return 1; - } - - qp_ind = find_qp(wc[i].qp_num, ctx, num_qp); - if (qp_ind < 0) { - fprintf(stderr, "Couldn't find QPN %06x\n", - wc[i].qp_num); - return 1; - } - - switch ((int) wc[i].wr_id) { - case PINGPONG_SEND_WRID: - ++scnt; - break; - - case PINGPONG_RECV_WRID: - if (--routs <= num_qp) { - routs += pp_post_recv(ctx, ctx->rx_depth - routs); - if (routs < ctx->rx_depth) { - fprintf(stderr, - "Couldn't post receive (%d)\n", - routs); - return 1; - } - } - - ++rcnt; - break; - - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - return 1; - } - - ctx->pending[qp_ind] &= ~(int) wc[i].wr_id; - if (scnt < iters && !ctx->pending[qp_ind]) { - if (pp_post_send(ctx, qp_ind)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending[qp_ind] = PINGPONG_RECV_WRID | - PINGPONG_SEND_WRID; - } - - } - } - } - - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - return 1; - } - - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - long long bytes = (long long) size * iters * 2; - - printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", - bytes, usec / 1000000., bytes * 8. / usec); - printf("%d iters in %.2f seconds = %.2f usec/iter\n", - iters, usec / 1000000., usec / iters); - } - - ibv_ack_cq_events(ctx->cq, num_cq_events); - - if (pp_close_ctx(ctx, num_qp)) - return 1; - - ibv_free_device_list(dev_list); - free(rem_dest); - - return 0; -} diff --git a/examples/uc_pingpong.c b/examples/uc_pingpong.c deleted file mode 100644 index 272dc26..0000000 --- a/examples/uc_pingpong.c +++ /dev/null @@ -1,857 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <netdb.h> -#include <malloc.h> -#include <getopt.h> -#include <arpa/inet.h> -#include <time.h> - -#include "pingpong.h" - -enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, -}; - -static int page_size; - -struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *cq; - struct ibv_qp *qp; - void *buf; - int size; - int send_flags; - int rx_depth; - int pending; - struct ibv_port_attr portinfo; -}; - -struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; -}; - -static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - enum ibv_mtu mtu, int sl, - struct pingpong_dest *dest, int sgid_idx) -{ - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR, - .path_mtu = mtu, - .dest_qp_num = dest->qpn, - .rq_psn = dest->psn, - .ah_attr = { - .is_global = 0, - .dlid = dest->lid, - .sl = sl, - .src_path_bits = 0, - .port_num = port - } - }; - - if (dest->gid.global.interface_id) { - attr.ah_attr.is_global = 1; - attr.ah_attr.grh.hop_limit = 1; - attr.ah_attr.grh.dgid = dest->gid; - attr.ah_attr.grh.sgid_index = sgid_idx; - } - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_AV | - IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | - IBV_QP_RQ_PSN)) { - fprintf(stderr, "Failed to modify QP to RTR\n"); - return 1; - } - - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = my_psn; - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_SQ_PSN)) { - fprintf(stderr, "Failed to modify QP to RTS\n"); - return 1; - } - - return 0; -} - -static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, - const struct pingpong_dest *my_dest) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(servername, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); - return NULL; - } - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(sockfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - goto out; - } - - if (read(sockfd, msg, sizeof msg) != sizeof msg) { - perror("client read"); - fprintf(stderr, "Couldn't read remote address\n"); - goto out; - } - - write(sockfd, "done", sizeof "done"); - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - -out: - close(sockfd); - return rem_dest; -} - -static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, - int ib_port, enum ibv_mtu mtu, - int port, int sl, - const struct pingpong_dest *my_dest, - int sgid_idx) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_flags = AI_PASSIVE, - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1, connfd; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(NULL, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - n = 1; - - setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); - - if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't listen to port %d\n", port); - return NULL; - } - - listen(sockfd, 1); - connfd = accept(sockfd, NULL, 0); - close(sockfd); - if (connfd < 0) { - fprintf(stderr, "accept() failed\n"); - return NULL; - } - - n = read(connfd, msg, sizeof msg); - if (n != sizeof msg) { - perror("server read"); - fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); - goto out; - } - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, - sgid_idx)) { - fprintf(stderr, "Couldn't connect to remote QP\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(connfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - read(connfd, msg, sizeof msg); - -out: - close(connfd); - return rem_dest; -} - -static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, - int rx_depth, int port, - int use_event) -{ - struct pingpong_context *ctx; - - ctx = calloc(1, sizeof *ctx); - if (!ctx) - return NULL; - - ctx->size = size; - ctx->send_flags = IBV_SEND_SIGNALED; - ctx->rx_depth = rx_depth; - - ctx->buf = memalign(page_size, size); - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } - - /* FIXME memset(ctx->buf, 0, size); */ - memset(ctx->buf, 0x7b, size); - - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); - if (!ctx->mr) { - fprintf(stderr, "Couldn't register MR\n"); - goto clean_pd; - } - - ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, - ctx->channel, 0); - if (!ctx->cq) { - fprintf(stderr, "Couldn't create CQ\n"); - goto clean_mr; - } - - { - struct ibv_qp_attr attr; - struct ibv_qp_init_attr init_attr = { - .send_cq = ctx->cq, - .recv_cq = ctx->cq, - .cap = { - .max_send_wr = 1, - .max_recv_wr = rx_depth, - .max_send_sge = 1, - .max_recv_sge = 1 - }, - .qp_type = IBV_QPT_UC - }; - - ctx->qp = ibv_create_qp(ctx->pd, &init_attr); - if (!ctx->qp) { - fprintf(stderr, "Couldn't create QP\n"); - goto clean_cq; - } - ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr); - if (init_attr.cap.max_inline_data >= size) { - ctx->send_flags |= IBV_SEND_INLINE; - } - } - - { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qp_access_flags = 0 - }; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_ACCESS_FLAGS)) { - fprintf(stderr, "Failed to modify QP to INIT\n"); - goto clean_qp; - } - } - - return ctx; - -clean_qp: - ibv_destroy_qp(ctx->qp); - -clean_cq: - ibv_destroy_cq(ctx->cq); - -clean_mr: - ibv_dereg_mr(ctx->mr); - -clean_pd: - ibv_dealloc_pd(ctx->pd); - -clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); - -clean_device: - ibv_close_device(ctx->context); - -clean_buffer: - free(ctx->buf); - -clean_ctx: - free(ctx); - - return NULL; -} - -int pp_close_ctx(struct pingpong_context *ctx) -{ - if (ibv_destroy_qp(ctx->qp)) { - fprintf(stderr, "Couldn't destroy QP\n"); - return 1; - } - - if (ibv_destroy_cq(ctx->cq)) { - fprintf(stderr, "Couldn't destroy CQ\n"); - return 1; - } - - if (ibv_dereg_mr(ctx->mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - return 1; - } - - if (ibv_dealloc_pd(ctx->pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - return 1; - } - - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - fprintf(stderr, "Couldn't destroy completion channel\n"); - return 1; - } - } - - if (ibv_close_device(ctx->context)) { - fprintf(stderr, "Couldn't release context\n"); - return 1; - } - - free(ctx->buf); - free(ctx); - - return 0; -} - -static int pp_post_recv(struct pingpong_context *ctx, int n) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; - - for (i = 0; i < n; ++i) - if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) - break; - - return i; -} - -static int pp_post_send(struct pingpong_context *ctx) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_send_wr wr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .opcode = IBV_WR_SEND, - .send_flags = ctx->send_flags, - }; - struct ibv_send_wr *bad_wr; - - return ibv_post_send(ctx->qp, &wr, &bad_wr); -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s <host> connect to server at <host>\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); - printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); - printf(" -m, --mtu=<size> path MTU (default 1024)\n"); - printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); - printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); - printf(" -l, --sl=<sl> service level value\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx=<gid index> local port gid index\n"); -} - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct pingpong_context *ctx; - struct pingpong_dest my_dest; - struct pingpong_dest *rem_dest; - struct timeval start, end; - char *ib_devname = NULL; - char *servername = NULL; - unsigned int port = 18515; - int ib_port = 1; - unsigned int size = 4096; - enum ibv_mtu mtu = IBV_MTU_1024; - unsigned int rx_depth = 500; - unsigned int iters = 1000; - int use_event = 0; - int routs; - int rcnt, scnt; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[33]; - - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "mtu", .has_arg = 1, .val = 'm' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", - long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtoul(optarg, NULL, 0); - if (port > 65535) { - usage(argv[0]); - return 1; - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 1) { - usage(argv[0]); - return 1; - } - break; - - case 's': - size = strtoul(optarg, NULL, 0); - break; - - case 'm': - mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); - if (mtu < 0) { - usage(argv[0]); - return 1; - } - break; - - case 'r': - rx_depth = strtoul(optarg, NULL, 0); - break; - - case 'n': - iters = strtoul(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - default: - usage(argv[0]); - return 1; - } - } - - if (optind == argc - 1) - servername = strdupa(argv[optind]); - else if (optind < argc) { - usage(argv[0]); - return 1; - } - - page_size = sysconf(_SC_PAGESIZE); - - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - - if (!ib_devname) { - ib_dev = *dev_list; - if (!ib_dev) { - fprintf(stderr, "No IB devices found\n"); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - fprintf(stderr, "IB device %s not found\n", ib_devname); - return 1; - } - } - - ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); - if (!ctx) - return 1; - - routs = pp_post_recv(ctx, ctx->rx_depth); - if (routs < ctx->rx_depth) { - fprintf(stderr, "Couldn't post receive (%d)\n", routs); - return 1; - } - - if (use_event) - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - fprintf(stderr, "Couldn't get port info\n"); - return 1; - } - - my_dest.lid = ctx->portinfo.lid; - if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && - !my_dest.lid) { - fprintf(stderr, "Couldn't get local LID\n"); - return 1; - } - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { - fprintf(stderr, "can't read sgid of index %d\n", gidx); - return 1; - } - } else - memset(&my_dest.gid, 0, sizeof my_dest.gid); - - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - my_dest.lid, my_dest.qpn, my_dest.psn, gid); - - - if (servername) - rem_dest = pp_client_exch_dest(servername, port, &my_dest); - else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, - &my_dest, gidx); - - if (!rem_dest) - return 1; - - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); - - if (servername) - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, - gidx)) - return 1; - - ctx->pending = PINGPONG_RECV_WRID; - - if (servername) { - if (pp_post_send(ctx)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending |= PINGPONG_SEND_WRID; - } - - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } - - rcnt = scnt = 0; - while (rcnt < iters || scnt < iters) { - if (use_event) { - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\n"); - return 1; - } - - ++num_cq_events; - - if (ev_cq != ctx->cq) { - fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - } - - { - struct ibv_wc wc[2]; - int ne, i; - - do { - ne = ibv_poll_cq(ctx->cq, 2, wc); - if (ne < 0) { - fprintf(stderr, "poll CQ failed %d\n", ne); - return 1; - } - - } while (!use_event && ne < 1); - - for (i = 0; i < ne; ++i) { - if (wc[i].status != IBV_WC_SUCCESS) { - fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); - return 1; - } - - switch ((int) wc[i].wr_id) { - case PINGPONG_SEND_WRID: - ++scnt; - break; - - case PINGPONG_RECV_WRID: - if (--routs <= 1) { - routs += pp_post_recv(ctx, ctx->rx_depth - routs); - if (routs < ctx->rx_depth) { - fprintf(stderr, - "Couldn't post receive (%d)\n", - routs); - return 1; - } - } - - ++rcnt; - break; - - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - return 1; - } - - ctx->pending &= ~(int) wc[i].wr_id; - if (scnt < iters && !ctx->pending) { - if (pp_post_send(ctx)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending = PINGPONG_RECV_WRID | - PINGPONG_SEND_WRID; - } - } - } - } - - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - return 1; - } - - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - long long bytes = (long long) size * iters * 2; - - printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", - bytes, usec / 1000000., bytes * 8. / usec); - printf("%d iters in %.2f seconds = %.2f usec/iter\n", - iters, usec / 1000000., usec / iters); - } - - ibv_ack_cq_events(ctx->cq, num_cq_events); - - if (pp_close_ctx(ctx)) - return 1; - - ibv_free_device_list(dev_list); - free(rem_dest); - - return 0; -} diff --git a/examples/ud_pingpong.c b/examples/ud_pingpong.c deleted file mode 100644 index 6d32ced..0000000 --- a/examples/ud_pingpong.c +++ /dev/null @@ -1,864 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <netdb.h> -#include <malloc.h> -#include <getopt.h> -#include <arpa/inet.h> -#include <time.h> - -#include "pingpong.h" - -enum { - PINGPONG_RECV_WRID = 1, - PINGPONG_SEND_WRID = 2, -}; - -static int page_size; - -struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *cq; - struct ibv_qp *qp; - struct ibv_ah *ah; - void *buf; - int size; - int send_flags; - int rx_depth; - int pending; - struct ibv_port_attr portinfo; -}; - -struct pingpong_dest { - int lid; - int qpn; - int psn; - union ibv_gid gid; -}; - -static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - int sl, struct pingpong_dest *dest, int sgid_idx) -{ - struct ibv_ah_attr ah_attr = { - .is_global = 0, - .dlid = dest->lid, - .sl = sl, - .src_path_bits = 0, - .port_num = port - }; - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_RTR - }; - - if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) { - fprintf(stderr, "Failed to modify QP to RTR\n"); - return 1; - } - - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = my_psn; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_SQ_PSN)) { - fprintf(stderr, "Failed to modify QP to RTS\n"); - return 1; - } - - if (dest->gid.global.interface_id) { - ah_attr.is_global = 1; - ah_attr.grh.hop_limit = 1; - ah_attr.grh.dgid = dest->gid; - ah_attr.grh.sgid_index = sgid_idx; - } - - ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); - if (!ctx->ah) { - fprintf(stderr, "Failed to create AH\n"); - return 1; - } - - return 0; -} - -static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, - const struct pingpong_dest *my_dest) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(servername, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); - return NULL; - } - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(sockfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - goto out; - } - - if (read(sockfd, msg, sizeof msg) != sizeof msg) { - perror("client read"); - fprintf(stderr, "Couldn't read remote address\n"); - goto out; - } - - write(sockfd, "done", sizeof "done"); - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - -out: - close(sockfd); - return rem_dest; -} - -static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, - int ib_port, int port, int sl, - const struct pingpong_dest *my_dest, - int sgid_idx) -{ - struct addrinfo *res, *t; - struct addrinfo hints = { - .ai_flags = AI_PASSIVE, - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - char *service; - char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; - int n; - int sockfd = -1, connfd; - struct pingpong_dest *rem_dest = NULL; - char gid[33]; - - if (asprintf(&service, "%d", port) < 0) - return NULL; - - n = getaddrinfo(NULL, service, &hints, &res); - - if (n < 0) { - fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); - free(service); - return NULL; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - n = 1; - - setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); - - if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't listen to port %d\n", port); - return NULL; - } - - listen(sockfd, 1); - connfd = accept(sockfd, NULL, 0); - close(sockfd); - if (connfd < 0) { - fprintf(stderr, "accept() failed\n"); - return NULL; - } - - n = read(connfd, msg, sizeof msg); - if (n != sizeof msg) { - perror("server read"); - fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); - goto out; - } - - rem_dest = malloc(sizeof *rem_dest); - if (!rem_dest) - goto out; - - sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, - &rem_dest->psn, gid); - wire_gid_to_gid(gid, &rem_dest->gid); - - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, - sgid_idx)) { - fprintf(stderr, "Couldn't connect to remote QP\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - gid_to_wire_gid(&my_dest->gid, gid); - sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, - my_dest->psn, gid); - if (write(connfd, msg, sizeof msg) != sizeof msg) { - fprintf(stderr, "Couldn't send local address\n"); - free(rem_dest); - rem_dest = NULL; - goto out; - } - - read(connfd, msg, sizeof msg); - -out: - close(connfd); - return rem_dest; -} - -static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, - int rx_depth, int port, - int use_event) -{ - struct pingpong_context *ctx; - - ctx = malloc(sizeof *ctx); - if (!ctx) - return NULL; - - ctx->size = size; - ctx->send_flags = IBV_SEND_SIGNALED; - ctx->rx_depth = rx_depth; - - ctx->buf = memalign(page_size, size + 40); - if (!ctx->buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - goto clean_ctx; - } - - /* FIXME memset(ctx->buf, 0, size + 40); */ - memset(ctx->buf, 0x7b, size + 40); - - ctx->context = ibv_open_device(ib_dev); - if (!ctx->context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(ib_dev)); - goto clean_buffer; - } - - { - struct ibv_port_attr port_info = { 0 }; - int mtu; - - if (ibv_query_port(ctx->context, port, &port_info)) { - fprintf(stderr, "Unable to query port info for port %d\n", port); - goto clean_device; - } - mtu = 1 << (port_info.active_mtu + 7); - if (size > mtu) { - fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu); - goto clean_device; - } - } - - if (use_event) { - ctx->channel = ibv_create_comp_channel(ctx->context); - if (!ctx->channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - goto clean_device; - } - } else - ctx->channel = NULL; - - ctx->pd = ibv_alloc_pd(ctx->context); - if (!ctx->pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - goto clean_comp_channel; - } - - ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE); - if (!ctx->mr) { - fprintf(stderr, "Couldn't register MR\n"); - goto clean_pd; - } - - ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, - ctx->channel, 0); - if (!ctx->cq) { - fprintf(stderr, "Couldn't create CQ\n"); - goto clean_mr; - } - - { - struct ibv_qp_attr attr; - struct ibv_qp_init_attr init_attr = { - .send_cq = ctx->cq, - .recv_cq = ctx->cq, - .cap = { - .max_send_wr = 1, - .max_recv_wr = rx_depth, - .max_send_sge = 1, - .max_recv_sge = 1 - }, - .qp_type = IBV_QPT_UD, - }; - - ctx->qp = ibv_create_qp(ctx->pd, &init_attr); - if (!ctx->qp) { - fprintf(stderr, "Couldn't create QP\n"); - goto clean_cq; - } - - ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr); - if (init_attr.cap.max_inline_data >= size) { - ctx->send_flags |= IBV_SEND_INLINE; - } - } - - { - struct ibv_qp_attr attr = { - .qp_state = IBV_QPS_INIT, - .pkey_index = 0, - .port_num = port, - .qkey = 0x11111111 - }; - - if (ibv_modify_qp(ctx->qp, &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_QKEY)) { - fprintf(stderr, "Failed to modify QP to INIT\n"); - goto clean_qp; - } - } - - return ctx; - -clean_qp: - ibv_destroy_qp(ctx->qp); - -clean_cq: - ibv_destroy_cq(ctx->cq); - -clean_mr: - ibv_dereg_mr(ctx->mr); - -clean_pd: - ibv_dealloc_pd(ctx->pd); - -clean_comp_channel: - if (ctx->channel) - ibv_destroy_comp_channel(ctx->channel); - -clean_device: - ibv_close_device(ctx->context); - -clean_buffer: - free(ctx->buf); - -clean_ctx: - free(ctx); - - return NULL; -} - -int pp_close_ctx(struct pingpong_context *ctx) -{ - if (ibv_destroy_qp(ctx->qp)) { - fprintf(stderr, "Couldn't destroy QP\n"); - return 1; - } - - if (ibv_destroy_cq(ctx->cq)) { - fprintf(stderr, "Couldn't destroy CQ\n"); - return 1; - } - - if (ibv_dereg_mr(ctx->mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - return 1; - } - - if (ibv_destroy_ah(ctx->ah)) { - fprintf(stderr, "Couldn't destroy AH\n"); - return 1; - } - - if (ibv_dealloc_pd(ctx->pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - return 1; - } - - if (ctx->channel) { - if (ibv_destroy_comp_channel(ctx->channel)) { - fprintf(stderr, "Couldn't destroy completion channel\n"); - return 1; - } - } - - if (ibv_close_device(ctx->context)) { - fprintf(stderr, "Couldn't release context\n"); - return 1; - } - - free(ctx->buf); - free(ctx); - - return 0; -} - -static int pp_post_recv(struct pingpong_context *ctx, int n) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf, - .length = ctx->size + 40, - .lkey = ctx->mr->lkey - }; - struct ibv_recv_wr wr = { - .wr_id = PINGPONG_RECV_WRID, - .sg_list = &list, - .num_sge = 1, - }; - struct ibv_recv_wr *bad_wr; - int i; - - for (i = 0; i < n; ++i) - if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) - break; - - return i; -} - -static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) -{ - struct ibv_sge list = { - .addr = (uintptr_t) ctx->buf + 40, - .length = ctx->size, - .lkey = ctx->mr->lkey - }; - struct ibv_send_wr wr = { - .wr_id = PINGPONG_SEND_WRID, - .sg_list = &list, - .num_sge = 1, - .opcode = IBV_WR_SEND, - .send_flags = ctx->send_flags, - .wr = { - .ud = { - .ah = ctx->ah, - .remote_qpn = qpn, - .remote_qkey = 0x11111111 - } - } - }; - struct ibv_send_wr *bad_wr; - - return ibv_post_send(ctx->qp, &wr, &bad_wr); -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s <host> connect to server at <host>\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); - printf(" -s, --size=<size> size of message to exchange (default 2048)\n"); - printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); - printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx=<gid index> local port gid index\n"); -} - -int main(int argc, char *argv[]) -{ - struct ibv_device **dev_list; - struct ibv_device *ib_dev; - struct pingpong_context *ctx; - struct pingpong_dest my_dest; - struct pingpong_dest *rem_dest; - struct timeval start, end; - char *ib_devname = NULL; - char *servername = NULL; - unsigned int port = 18515; - int ib_port = 1; - unsigned int size = 2048; - unsigned int rx_depth = 500; - unsigned int iters = 1000; - int use_event = 0; - int routs; - int rcnt, scnt; - int num_cq_events = 0; - int sl = 0; - int gidx = -1; - char gid[33]; - - srand48(getpid() * time(NULL)); - - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "rx-depth", .has_arg = 1, .val = 'r' }, - { .name = "iters", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", - long_options, NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtol(optarg, NULL, 0); - if (port > 65535) { - usage(argv[0]); - return 1; - } - break; - - case 'd': - ib_devname = strdupa(optarg); - break; - - case 'i': - ib_port = strtol(optarg, NULL, 0); - if (ib_port < 1) { - usage(argv[0]); - return 1; - } - break; - - case 's': - size = strtoul(optarg, NULL, 0); - break; - - case 'r': - rx_depth = strtoul(optarg, NULL, 0); - break; - - case 'n': - iters = strtoul(optarg, NULL, 0); - break; - - case 'l': - sl = strtol(optarg, NULL, 0); - break; - - case 'e': - ++use_event; - break; - - case 'g': - gidx = strtol(optarg, NULL, 0); - break; - - default: - usage(argv[0]); - return 1; - } - } - - if (optind == argc - 1) - servername = strdupa(argv[optind]); - else if (optind < argc) { - usage(argv[0]); - return 1; - } - - page_size = sysconf(_SC_PAGESIZE); - - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - perror("Failed to get IB devices list"); - return 1; - } - - if (!ib_devname) { - ib_dev = *dev_list; - if (!ib_dev) { - fprintf(stderr, "No IB devices found\n"); - return 1; - } - } else { - int i; - for (i = 0; dev_list[i]; ++i) - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - ib_dev = dev_list[i]; - if (!ib_dev) { - fprintf(stderr, "IB device %s not found\n", ib_devname); - return 1; - } - } - - ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); - if (!ctx) - return 1; - - routs = pp_post_recv(ctx, ctx->rx_depth); - if (routs < ctx->rx_depth) { - fprintf(stderr, "Couldn't post receive (%d)\n", routs); - return 1; - } - - if (use_event) - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - - if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { - fprintf(stderr, "Couldn't get port info\n"); - return 1; - } - my_dest.lid = ctx->portinfo.lid; - - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - - if (gidx >= 0) { - if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { - fprintf(stderr, "Could not get local gid for gid index " - "%d\n", gidx); - return 1; - } - } else - memset(&my_dest.gid, 0, sizeof my_dest.gid); - - inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", - my_dest.lid, my_dest.qpn, my_dest.psn, gid); - - if (servername) - rem_dest = pp_client_exch_dest(servername, port, &my_dest); - else - rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, - &my_dest, gidx); - - if (!rem_dest) - return 1; - - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); - - if (servername) - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, - gidx)) - return 1; - - ctx->pending = PINGPONG_RECV_WRID; - - if (servername) { - if (pp_post_send(ctx, rem_dest->qpn)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending |= PINGPONG_SEND_WRID; - } - - if (gettimeofday(&start, NULL)) { - perror("gettimeofday"); - return 1; - } - - rcnt = scnt = 0; - while (rcnt < iters || scnt < iters) { - if (use_event) { - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\n"); - return 1; - } - - ++num_cq_events; - - if (ev_cq != ctx->cq) { - fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(ctx->cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - } - - { - struct ibv_wc wc[2]; - int ne, i; - - do { - ne = ibv_poll_cq(ctx->cq, 2, wc); - if (ne < 0) { - fprintf(stderr, "poll CQ failed %d\n", ne); - return 1; - } - } while (!use_event && ne < 1); - - for (i = 0; i < ne; ++i) { - if (wc[i].status != IBV_WC_SUCCESS) { - fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", - ibv_wc_status_str(wc[i].status), - wc[i].status, (int) wc[i].wr_id); - return 1; - } - - switch ((int) wc[i].wr_id) { - case PINGPONG_SEND_WRID: - ++scnt; - break; - - case PINGPONG_RECV_WRID: - if (--routs <= 1) { - routs += pp_post_recv(ctx, ctx->rx_depth - routs); - if (routs < ctx->rx_depth) { - fprintf(stderr, - "Couldn't post receive (%d)\n", - routs); - return 1; - } - } - - ++rcnt; - break; - - default: - fprintf(stderr, "Completion for unknown wr_id %d\n", - (int) wc[i].wr_id); - return 1; - } - - ctx->pending &= ~(int) wc[i].wr_id; - if (scnt < iters && !ctx->pending) { - if (pp_post_send(ctx, rem_dest->qpn)) { - fprintf(stderr, "Couldn't post send\n"); - return 1; - } - ctx->pending = PINGPONG_RECV_WRID | - PINGPONG_SEND_WRID; - } - } - } - } - - if (gettimeofday(&end, NULL)) { - perror("gettimeofday"); - return 1; - } - - { - float usec = (end.tv_sec - start.tv_sec) * 1000000 + - (end.tv_usec - start.tv_usec); - long long bytes = (long long) size * iters * 2; - - printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", - bytes, usec / 1000000., bytes * 8. / usec); - printf("%d iters in %.2f seconds = %.2f usec/iter\n", - iters, usec / 1000000., usec / iters); - } - - ibv_ack_cq_events(ctx->cq, num_cq_events); - - if (pp_close_ctx(ctx)) - return 1; - - ibv_free_device_list(dev_list); - free(rem_dest); - - return 0; -} diff --git a/examples/xsrq_pingpong.c b/examples/xsrq_pingpong.c deleted file mode 100644 index 70c5760..0000000 --- a/examples/xsrq_pingpong.c +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2011 Intel Corporation, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <fcntl.h> -#include <errno.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <netdb.h> -#include <malloc.h> -#include <getopt.h> -#include <arpa/inet.h> -#include <time.h> - -#include "pingpong.h" - -#define MSG_FORMAT "%04x:%06x:%06x:%06x:%06x:%32s" -#define MSG_SIZE 66 -#define MSG_SSCAN "%x:%x:%x:%x:%x:%s" -#define ADDR_FORMAT \ - "%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %06x, GID %s\n" -#define TERMINATION_FORMAT "%s" -#define TERMINATION_MSG_SIZE 4 -#define TERMINATION_MSG "END" -static int page_size; - -struct pingpong_dest { - union ibv_gid gid; - int lid; - int recv_qpn; - int send_qpn; - int recv_psn; - int send_psn; - int srqn; - int pp_cnt; - int sockfd; -}; - -struct pingpong_context { - struct ibv_context *context; - struct ibv_comp_channel *channel; - struct ibv_pd *pd; - struct ibv_mr *mr; - struct ibv_cq *send_cq; - struct ibv_cq *recv_cq; - struct ibv_srq *srq; - struct ibv_xrcd *xrcd; - struct ibv_qp **recv_qp; - struct ibv_qp **send_qp; - struct pingpong_dest *rem_dest; - void *buf; - int lid; - int sl; - enum ibv_mtu mtu; - int ib_port; - int fd; - int size; - int num_clients; - int num_tests; - int use_event; - int gidx; -}; - -struct pingpong_context ctx; - - -static int open_device(char *ib_devname) -{ - struct ibv_device **dev_list; - int i = 0; - - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - fprintf(stderr, "Failed to get IB devices list"); - return -1; - } - - if (ib_devname) { - for (; dev_list[i]; ++i) { - if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) - break; - } - } - if (!dev_list[i]) { - fprintf(stderr, "IB device %s not found\n", - ib_devname ? ib_devname : ""); - return -1; - } - - ctx.context = ibv_open_device(dev_list[i]); - if (!ctx.context) { - fprintf(stderr, "Couldn't get context for %s\n", - ibv_get_device_name(dev_list[i])); - return -1; - } - - ibv_free_device_list(dev_list); - return 0; -} - -static int create_qps(void) -{ - struct ibv_qp_init_attr_ex init; - struct ibv_qp_attr mod; - int i; - - for (i = 0; i < ctx.num_clients; ++i) { - - memset(&init, 0, sizeof init); - init.qp_type = IBV_QPT_XRC_RECV; - init.comp_mask = IBV_QP_INIT_ATTR_XRCD; - init.xrcd = ctx.xrcd; - - ctx.recv_qp[i] = ibv_create_qp_ex(ctx.context, &init); - if (!ctx.recv_qp[i]) { - fprintf(stderr, "Couldn't create recv QP[%d] errno %d\n", - i, errno); - return 1; - } - - mod.qp_state = IBV_QPS_INIT; - mod.pkey_index = 0; - mod.port_num = ctx.ib_port; - mod.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; - - if (ibv_modify_qp(ctx.recv_qp[i], &mod, - IBV_QP_STATE | IBV_QP_PKEY_INDEX | - IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) { - fprintf(stderr, "Failed to modify recv QP[%d] to INIT\n", i); - return 1; - } - - memset(&init, 0, sizeof init); - init.qp_type = IBV_QPT_XRC_SEND; - init.send_cq = ctx.send_cq; - init.cap.max_send_wr = ctx.num_clients * ctx.num_tests; - init.cap.max_send_sge = 1; - init.comp_mask = IBV_QP_INIT_ATTR_PD; - init.pd = ctx.pd; - - ctx.send_qp[i] = ibv_create_qp_ex(ctx.context, &init); - if (!ctx.send_qp[i]) { - fprintf(stderr, "Couldn't create send QP[%d] errno %d\n", - i, errno); - return 1; - } - - mod.qp_state = IBV_QPS_INIT; - mod.pkey_index = 0; - mod.port_num = ctx.ib_port; - mod.qp_access_flags = 0; - - if (ibv_modify_qp(ctx.send_qp[i], &mod, - IBV_QP_STATE | IBV_QP_PKEY_INDEX | - IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) { - fprintf(stderr, "Failed to modify send QP[%d] to INIT\n", i); - return 1; - } - } - - return 0; -} - -static int pp_init_ctx(char *ib_devname) -{ - struct ibv_srq_init_attr_ex attr; - struct ibv_xrcd_init_attr xrcd_attr; - struct ibv_port_attr port_attr; - - ctx.recv_qp = calloc(ctx.num_clients, sizeof *ctx.recv_qp); - ctx.send_qp = calloc(ctx.num_clients, sizeof *ctx.send_qp); - ctx.rem_dest = calloc(ctx.num_clients, sizeof *ctx.rem_dest); - if (!ctx.recv_qp || !ctx.send_qp || !ctx.rem_dest) - return 1; - - if (open_device(ib_devname)) { - fprintf(stderr, "Failed to open device\n"); - return 1; - } - - if (pp_get_port_info(ctx.context, ctx.ib_port, &port_attr)) { - fprintf(stderr, "Failed to get port info\n"); - return 1; - } - - ctx.lid = port_attr.lid; - if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET && !ctx.lid) { - fprintf(stderr, "Couldn't get local LID\n"); - return 1; - } - - ctx.buf = memalign(page_size, ctx.size); - if (!ctx.buf) { - fprintf(stderr, "Couldn't allocate work buf.\n"); - return 1; - } - - memset(ctx.buf, 0, ctx.size); - - if (ctx.use_event) { - ctx.channel = ibv_create_comp_channel(ctx.context); - if (!ctx.channel) { - fprintf(stderr, "Couldn't create completion channel\n"); - return 1; - } - } - - ctx.pd = ibv_alloc_pd(ctx.context); - if (!ctx.pd) { - fprintf(stderr, "Couldn't allocate PD\n"); - return 1; - } - - ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, ctx.size, IBV_ACCESS_LOCAL_WRITE); - if (!ctx.mr) { - fprintf(stderr, "Couldn't register MR\n"); - return 1; - } - - ctx.fd = open("/tmp/xrc_domain", O_RDONLY | O_CREAT, S_IRUSR | S_IRGRP); - if (ctx.fd < 0) { - fprintf(stderr, - "Couldn't create the file for the XRC Domain " - "but not stopping %d\n", errno); - ctx.fd = -1; - } - - memset(&xrcd_attr, 0, sizeof xrcd_attr); - xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS; - xrcd_attr.fd = ctx.fd; - xrcd_attr.oflags = O_CREAT; - ctx.xrcd = ibv_open_xrcd(ctx.context, &xrcd_attr); - if (!ctx.xrcd) { - fprintf(stderr, "Couldn't Open the XRC Domain %d\n", errno); - return 1; - } - - ctx.recv_cq = ibv_create_cq(ctx.context, ctx.num_clients, &ctx.recv_cq, - ctx.channel, 0); - if (!ctx.recv_cq) { - fprintf(stderr, "Couldn't create recv CQ\n"); - return 1; - } - - if (ctx.use_event) { - if (ibv_req_notify_cq(ctx.recv_cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - } - - ctx.send_cq = ibv_create_cq(ctx.context, ctx.num_clients, NULL, NULL, 0); - if (!ctx.send_cq) { - fprintf(stderr, "Couldn't create send CQ\n"); - return 1; - } - - memset(&attr, 0, sizeof attr); - attr.attr.max_wr = ctx.num_clients; - attr.attr.max_sge = 1; - attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_XRCD | - IBV_SRQ_INIT_ATTR_CQ | IBV_SRQ_INIT_ATTR_PD; - attr.srq_type = IBV_SRQT_XRC; - attr.xrcd = ctx.xrcd; - attr.cq = ctx.recv_cq; - attr.pd = ctx.pd; - - ctx.srq = ibv_create_srq_ex(ctx.context, &attr); - if (!ctx.srq) { - fprintf(stderr, "Couldn't create SRQ\n"); - return 1; - } - - if (create_qps()) - return 1; - - return 0; -} - -static int recv_termination_ack(int index) -{ - char msg[TERMINATION_MSG_SIZE]; - int n = 0, r; - int sockfd = ctx.rem_dest[index].sockfd; - - while (n < TERMINATION_MSG_SIZE) { - r = read(sockfd, msg + n, TERMINATION_MSG_SIZE - n); - if (r < 0) { - perror("client read"); - fprintf(stderr, - "%d/%d: Couldn't read remote termination ack\n", - n, TERMINATION_MSG_SIZE); - return 1; - } - n += r; - } - - if (strcmp(msg, TERMINATION_MSG)) { - fprintf(stderr, "Invalid termination ack was accepted\n"); - return 1; - } - - return 0; -} - -static int send_termination_ack(int index) -{ - char msg[TERMINATION_MSG_SIZE]; - int sockfd = ctx.rem_dest[index].sockfd; - - sprintf(msg, TERMINATION_FORMAT, TERMINATION_MSG); - - if (write(sockfd, msg, TERMINATION_MSG_SIZE) != TERMINATION_MSG_SIZE) { - fprintf(stderr, "Couldn't send termination ack\n"); - return 1; - } - - return 0; -} - -static int pp_client_termination() -{ - if (send_termination_ack(0)) - return 1; - if (recv_termination_ack(0)) - return 1; - - return 0; -} - -static int pp_server_termination() -{ - int i; - - for (i = 0; i < ctx.num_clients; i++) { - if (recv_termination_ack(i)) - return 1; - } - - for (i = 0; i < ctx.num_clients; i++) { - if (send_termination_ack(i)) - return 1; - } - - return 0; -} - -static int send_local_dest(int sockfd, int index) -{ - char msg[MSG_SIZE]; - char gid[33]; - uint32_t srq_num; - union ibv_gid local_gid; - - if (ctx.gidx >= 0) { - if (ibv_query_gid(ctx.context, ctx.ib_port, ctx.gidx, - &local_gid)) { - fprintf(stderr, "can't read sgid of index %d\n", - ctx.gidx); - return -1; - } - } else { - memset(&local_gid, 0, sizeof(local_gid)); - } - - ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff; - if (ibv_get_srq_num(ctx.srq, &srq_num)) { - fprintf(stderr, "Couldn't get SRQ num\n"); - return -1; - } - - inet_ntop(AF_INET6, &local_gid, gid, sizeof(gid)); - printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num, - ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn, - srq_num, gid); - - gid_to_wire_gid(&local_gid, gid); - sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num, - ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn, - srq_num, gid); - - if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) { - fprintf(stderr, "Couldn't send local address\n"); - return -1; - } - - return 0; -} - -static int recv_remote_dest(int sockfd, int index) -{ - struct pingpong_dest *rem_dest; - char msg[MSG_SIZE]; - char gid[33]; - int n = 0, r; - - while (n < MSG_SIZE) { - r = read(sockfd, msg + n, MSG_SIZE - n); - if (r < 0) { - perror("client read"); - fprintf(stderr, - "%d/%d: Couldn't read remote address [%d]\n", - n, MSG_SIZE, index); - return -1; - } - n += r; - } - - rem_dest = &ctx.rem_dest[index]; - sscanf(msg, MSG_SSCAN, &rem_dest->lid, &rem_dest->recv_qpn, - &rem_dest->send_qpn, &rem_dest->send_psn, &rem_dest->srqn, gid); - - wire_gid_to_gid(gid, &rem_dest->gid); - inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof(gid)); - printf(ADDR_FORMAT, "remote", rem_dest->lid, rem_dest->recv_qpn, - rem_dest->send_qpn, rem_dest->send_psn, rem_dest->srqn, - gid); - - rem_dest->sockfd = sockfd; - return 0; -} - -static void set_ah_attr(struct ibv_ah_attr *attr, struct pingpong_context *ctx, - int index) -{ - attr->is_global = 1; - attr->grh.hop_limit = 5; - attr->grh.dgid = ctx->rem_dest[index].gid; - attr->grh.sgid_index = ctx->gidx; -} - -static int connect_qps(int index) -{ - struct ibv_qp_attr attr; - - memset(&attr, 0, sizeof attr); - attr.qp_state = IBV_QPS_RTR; - attr.dest_qp_num = ctx.rem_dest[index].send_qpn; - attr.path_mtu = ctx.mtu; - attr.rq_psn = ctx.rem_dest[index].send_psn; - attr.min_rnr_timer = 12; - attr.ah_attr.dlid = ctx.rem_dest[index].lid; - attr.ah_attr.sl = ctx.sl; - attr.ah_attr.port_num = ctx.ib_port; - - if (ctx.rem_dest[index].gid.global.interface_id) - set_ah_attr(&attr.ah_attr, &ctx, index); - - if (ibv_modify_qp(ctx.recv_qp[index], &attr, - IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | - IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) { - fprintf(stderr, "Failed to modify recv QP[%d] to RTR\n", index); - return 1; - } - - memset(&attr, 0, sizeof attr); - attr.qp_state = IBV_QPS_RTS; - attr.timeout = 14; - attr.sq_psn = ctx.rem_dest[index].recv_psn; - - if (ibv_modify_qp(ctx.recv_qp[index], &attr, - IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN)) { - fprintf(stderr, "Failed to modify recv QP[%d] to RTS\n", index); - return 1; - } - - memset(&attr, 0, sizeof attr); - attr.qp_state = IBV_QPS_RTR; - attr.dest_qp_num = ctx.rem_dest[index].recv_qpn; - attr.path_mtu = ctx.mtu; - attr.rq_psn = ctx.rem_dest[index].send_psn; - attr.ah_attr.dlid = ctx.rem_dest[index].lid; - attr.ah_attr.sl = ctx.sl; - attr.ah_attr.port_num = ctx.ib_port; - - if (ctx.rem_dest[index].gid.global.interface_id) - set_ah_attr(&attr.ah_attr, &ctx, index); - - if (ibv_modify_qp(ctx.send_qp[index], &attr, - IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | IBV_QP_RQ_PSN)) { - fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index); - return 1; - } - - memset(&attr, 0, sizeof attr); - attr.qp_state = IBV_QPS_RTS; - attr.timeout = 14; - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.sq_psn = ctx.rem_dest[index].recv_psn; - - if (ibv_modify_qp(ctx.send_qp[index], &attr, - IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN | - IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) { - fprintf(stderr, "Failed to modify send QP[%d] to RTS\n", index); - return 1; - } - - return 0; -} - -static int pp_client_connect(const char *servername, int port) -{ - struct addrinfo *res, *t; - char *service; - int ret; - int sockfd = -1; - struct addrinfo hints = { - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - - if (asprintf(&service, "%d", port) < 0) - return 1; - - ret = getaddrinfo(servername, service, &hints, &res); - if (ret < 0) { - fprintf(stderr, "%s for %s:%d\n", gai_strerror(ret), servername, port); - free(service); - return 1; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); - return 1; - } - - if (send_local_dest(sockfd, 0)) - return 1; - - if (recv_remote_dest(sockfd, 0)) - return 1; - - if (connect_qps(0)) - return 1; - - return 0; -} - -static int pp_server_connect(int port) -{ - struct addrinfo *res, *t; - char *service; - int ret, i, n; - int sockfd = -1, connfd; - struct addrinfo hints = { - .ai_flags = AI_PASSIVE, - .ai_family = AF_UNSPEC, - .ai_socktype = SOCK_STREAM - }; - - if (asprintf(&service, "%d", port) < 0) - return 1; - - ret = getaddrinfo(NULL, service, &hints, &res); - if (ret < 0) { - fprintf(stderr, "%s for port %d\n", gai_strerror(ret), port); - free(service); - return 1; - } - - for (t = res; t; t = t->ai_next) { - sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); - if (sockfd >= 0) { - n = 1; - setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); - if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) - break; - close(sockfd); - sockfd = -1; - } - } - - freeaddrinfo(res); - free(service); - - if (sockfd < 0) { - fprintf(stderr, "Couldn't listen to port %d\n", port); - return 1; - } - - listen(sockfd, ctx.num_clients); - - for (i = 0; i < ctx.num_clients; i++) { - connfd = accept(sockfd, NULL, 0); - if (connfd < 0) { - fprintf(stderr, "accept() failed for client %d\n", i); - return 1; - } - - if (recv_remote_dest(connfd, i)) - return 1; - - if (send_local_dest(connfd, i)) - return 1; - - if (connect_qps(i)) - return 1; - } - - close(sockfd); - return 0; -} - - -static int pp_close_ctx(void) -{ - int i; - - for (i = 0; i < ctx.num_clients; ++i) { - - if (ibv_destroy_qp(ctx.send_qp[i])) { - fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i); - return 1; - } - - if (ibv_destroy_qp(ctx.recv_qp[i])) { - fprintf(stderr, "Couldn't destroy TGT QP[%d]\n", i); - return 1; - } - - if (ctx.rem_dest[i].sockfd) - close(ctx.rem_dest[i].sockfd); - } - - if (ibv_destroy_srq(ctx.srq)) { - fprintf(stderr, "Couldn't destroy SRQ\n"); - return 1; - } - - if (ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) { - fprintf(stderr, "Couldn't close the XRC Domain\n"); - return 1; - } - if (ctx.fd >= 0 && close(ctx.fd)) { - fprintf(stderr, "Couldn't close the file for the XRC Domain\n"); - return 1; - } - - if (ibv_destroy_cq(ctx.send_cq)) { - fprintf(stderr, "Couldn't destroy send CQ\n"); - return 1; - } - - if (ibv_destroy_cq(ctx.recv_cq)) { - fprintf(stderr, "Couldn't destroy recv CQ\n"); - return 1; - } - - if (ibv_dereg_mr(ctx.mr)) { - fprintf(stderr, "Couldn't deregister MR\n"); - return 1; - } - - if (ibv_dealloc_pd(ctx.pd)) { - fprintf(stderr, "Couldn't deallocate PD\n"); - return 1; - } - - if (ctx.channel) { - if (ibv_destroy_comp_channel(ctx.channel)) { - fprintf(stderr, - "Couldn't destroy completion channel\n"); - return 1; - } - } - - if (ibv_close_device(ctx.context)) { - fprintf(stderr, "Couldn't release context\n"); - return 1; - } - - free(ctx.buf); - free(ctx.rem_dest); - free(ctx.send_qp); - free(ctx.recv_qp); - return 0; -} - -static int pp_post_recv(int cnt) -{ - struct ibv_sge sge; - struct ibv_recv_wr wr, *bad_wr; - - sge.addr = (uintptr_t) ctx.buf; - sge.length = ctx.size; - sge.lkey = ctx.mr->lkey; - - wr.next = NULL; - wr.wr_id = (uintptr_t) &ctx; - wr.sg_list = &sge; - wr.num_sge = 1; - - while (cnt--) { - if (ibv_post_srq_recv(ctx.srq, &wr, &bad_wr)) { - fprintf(stderr, "Failed to post receive to SRQ\n"); - return 1; - } - } - return 0; -} - -/* - * Send to each client round robin on each set of xrc send/recv qp. - * Generate a completion on the last send. - */ -static int pp_post_send(int index) -{ - struct ibv_sge sge; - struct ibv_send_wr wr, *bad_wr; - int qpi; - - sge.addr = (uintptr_t) ctx.buf; - sge.length = ctx.size; - sge.lkey = ctx.mr->lkey; - - wr.wr_id = (uintptr_t) index; - wr.next = NULL; - wr.sg_list = &sge; - wr.num_sge = 1; - wr.opcode = IBV_WR_SEND; - wr.qp_type.xrc.remote_srqn = ctx.rem_dest[index].srqn; - - qpi = (index + ctx.rem_dest[index].pp_cnt) % ctx.num_clients; - wr.send_flags = (++ctx.rem_dest[index].pp_cnt >= ctx.num_tests) ? - IBV_SEND_SIGNALED : 0; - - return ibv_post_send(ctx.send_qp[qpi], &wr, &bad_wr); -} - -static int find_qp(int qpn) -{ - int i; - - if (ctx.num_clients == 1) - return 0; - - for (i = 0; i < ctx.num_clients; ++i) - if (ctx.recv_qp[i]->qp_num == qpn) - return i; - - fprintf(stderr, "Unable to find qp %x\n", qpn); - return 0; -} - -static int get_cq_event(void) -{ - struct ibv_cq *ev_cq; - void *ev_ctx; - - if (ibv_get_cq_event(ctx.channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\n"); - return 1; - } - - if (ev_cq != ctx.recv_cq) { - fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); - return 1; - } - - if (ibv_req_notify_cq(ctx.recv_cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\n"); - return 1; - } - - return 0; -} - -static void init(void) -{ - srand48(getpid() * time(NULL)); - - ctx.size = 4096; - ctx.ib_port = 1; - ctx.num_clients = 1; - ctx.num_tests = 5; - ctx.mtu = IBV_MTU_2048; - ctx.sl = 0; - ctx.gidx = -1; -} - -static void usage(const char *argv0) -{ - printf("Usage:\n"); - printf(" %s start a server and wait for connection\n", argv0); - printf(" %s <host> connect to server at <host>\n", argv0); - printf("\n"); - printf("Options:\n"); - printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); - printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); - printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); - printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); - printf(" -m, --mtu=<size> path MTU (default 2048)\n"); - printf(" -c, --clients=<n> number of clients (on server only, default 1)\n"); - printf(" -n, --num_tests=<n> number of tests per client (default 5)\n"); - printf(" -l, --sl=<sl> service level value\n"); - printf(" -e, --events sleep on CQ events (default poll)\n"); - printf(" -g, --gid-idx=<gid index> local port gid index\n"); -} - -int main(int argc, char *argv[]) -{ - char *ib_devname = NULL; - char *servername = NULL; - int port = 18515; - int i, total, cnt = 0; - int ne, qpi, num_cq_events = 0; - struct ibv_wc wc; - - init(); - while (1) { - int c; - - static struct option long_options[] = { - { .name = "port", .has_arg = 1, .val = 'p' }, - { .name = "ib-dev", .has_arg = 1, .val = 'd' }, - { .name = "ib-port", .has_arg = 1, .val = 'i' }, - { .name = "size", .has_arg = 1, .val = 's' }, - { .name = "mtu", .has_arg = 1, .val = 'm' }, - { .name = "clients", .has_arg = 1, .val = 'c' }, - { .name = "num_tests", .has_arg = 1, .val = 'n' }, - { .name = "sl", .has_arg = 1, .val = 'l' }, - { .name = "events", .has_arg = 0, .val = 'e' }, - { .name = "gid-idx", .has_arg = 1, .val = 'g' }, - { 0 } - }; - - c = getopt_long(argc, argv, "p:d:i:s:m:c:n:l:eg:", long_options, - NULL); - if (c == -1) - break; - - switch (c) { - case 'p': - port = strtol(optarg, NULL, 0); - if (port < 0 || port > 65535) { - usage(argv[0]); - return 1; - } - break; - case 'd': - ib_devname = strdupa(optarg); - break; - case 'i': - ctx.ib_port = strtol(optarg, NULL, 0); - if (ctx.ib_port < 0) { - usage(argv[0]); - return 1; - } - break; - case 's': - ctx.size = strtol(optarg, NULL, 0); - break; - case 'm': - ctx.mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); - if (ctx.mtu < 0) { - usage(argv[0]); - return 1; - } - break; - case 'c': - ctx.num_clients = strtol(optarg, NULL, 0); - break; - case 'n': - ctx.num_tests = strtol(optarg, NULL, 0); - break; - case 'l': - ctx.sl = strtol(optarg, NULL, 0); - break; - case 'g': - ctx.gidx = strtol(optarg, NULL, 0); - break; - case 'e': - ctx.use_event = 1; - break; - default: - usage(argv[0]); - return 1; - } - } - - if (optind == argc - 1) { - servername = strdupa(argv[optind]); - ctx.num_clients = 1; - } else if (optind < argc) { - usage(argv[0]); - return 1; - } - - page_size = sysconf(_SC_PAGESIZE); - - if (pp_init_ctx(ib_devname)) - return 1; - - if (pp_post_recv(ctx.num_clients)) { - fprintf(stderr, "Couldn't post receives\n"); - return 1; - } - - if (servername) { - if (pp_client_connect(servername, port)) - return 1; - } else { - if (pp_server_connect(port)) - return 1; - - for (i = 0; i < ctx.num_clients; i++) - pp_post_send(i); - } - - total = ctx.num_clients * ctx.num_tests; - while (cnt < total) { - if (ctx.use_event) { - if (get_cq_event()) - return 1; - - ++num_cq_events; - } - - do { - ne = ibv_poll_cq(ctx.recv_cq, 1, &wc); - if (ne < 0) { - fprintf(stderr, "Error polling cq %d\n", ne); - return 1; - } else if (ne == 0) { - break; - } - - if (wc.status) { - fprintf(stderr, "Work completion error %d\n", wc.status); - return 1; - } - - pp_post_recv(ne); - qpi = find_qp(wc.qp_num); - if (ctx.rem_dest[qpi].pp_cnt < ctx.num_tests) - pp_post_send(qpi); - cnt += ne; - } while (ne > 0); - } - - for (cnt = 0; cnt < ctx.num_clients; cnt += ne) { - ne = ibv_poll_cq(ctx.send_cq, 1, &wc); - if (ne < 0) { - fprintf(stderr, "Error polling cq %d\n", ne); - return 1; - } - } - - if (ctx.use_event) - ibv_ack_cq_events(ctx.recv_cq, num_cq_events); - - /* Process should get an ack from the daemon to close its resources to - * make sure latest daemon's response sent via its target QP destined - * to an XSRQ created by another client won't be lost. - * Failure to do so may cause the other client to wait for that sent - * message forever. See comment on pp_post_send. - */ - if (servername) { - if (pp_client_termination()) - return 1; - } else if (pp_server_termination()) { - return 1; - } - - if (pp_close_ctx()) - return 1; - - printf("success\n"); - return 0; -} diff --git a/include/infiniband/arch.h b/include/infiniband/arch.h deleted file mode 100644 index e35ecf0..0000000 --- a/include/infiniband/arch.h +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_ARCH_H -#define INFINIBAND_ARCH_H - -#include <stdint.h> -#include <endian.h> -#include <byteswap.h> - -#if __BYTE_ORDER == __LITTLE_ENDIAN -static inline uint64_t htonll(uint64_t x) { return bswap_64(x); } -static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); } -#elif __BYTE_ORDER == __BIG_ENDIAN -static inline uint64_t htonll(uint64_t x) { return x; } -static inline uint64_t ntohll(uint64_t x) { return x; } -#else -#error __BYTE_ORDER is neither __LITTLE_ENDIAN nor __BIG_ENDIAN -#endif - -/* - * Architecture-specific defines. Currently, an architecture is - * required to implement the following operations: - * - * mb() - memory barrier. No loads or stores may be reordered across - * this macro by either the compiler or the CPU. - * rmb() - read memory barrier. No loads may be reordered across this - * macro by either the compiler or the CPU. - * wmb() - write memory barrier. No stores may be reordered across - * this macro by either the compiler or the CPU. - * wc_wmb() - flush write combine buffers. No write-combined writes - * will be reordered across this macro by either the compiler or - * the CPU. - */ - -#if defined(__i386__) - -#define mb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") -#define rmb() mb() -#define wmb() asm volatile("" ::: "memory") -#define wc_wmb() mb() - -#elif defined(__x86_64__) - -/* - * Only use lfence for mb() and rmb() because we don't care about - * ordering against non-temporal stores (for now at least). - */ -#define mb() asm volatile("lfence" ::: "memory") -#define rmb() mb() -#define wmb() asm volatile("" ::: "memory") -#define wc_wmb() asm volatile("sfence" ::: "memory") - -#elif defined(__PPC64__) - -#define mb() asm volatile("sync" ::: "memory") -#define rmb() asm volatile("lwsync" ::: "memory") -#define wmb() mb() -#define wc_wmb() wmb() - -#elif defined(__ia64__) - -#define mb() asm volatile("mf" ::: "memory") -#define rmb() mb() -#define wmb() mb() -#define wc_wmb() asm volatile("fwb" ::: "memory") - -#elif defined(__PPC__) - -#define mb() asm volatile("sync" ::: "memory") -#define rmb() mb() -#define wmb() mb() -#define wc_wmb() wmb() - -#elif defined(__sparc_v9__) - -#define mb() asm volatile("membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad" ::: "memory") -#define rmb() asm volatile("membar #LoadLoad" ::: "memory") -#define wmb() asm volatile("membar #StoreStore" ::: "memory") -#define wc_wmb() wmb() - -#elif defined(__sparc__) - -#define mb() asm volatile("" ::: "memory") -#define rmb() mb() -#define wmb() mb() -#define wc_wmb() wmb() - -#elif defined(__s390x__) - -#define mb() { asm volatile("" : : : "memory"); } /* for s390x */ -#define rmb() mb() /* for s390x */ -#define wmb() mb() /* for s390x */ -#define wc_wmb() wmb() /* for s390x */ - -#elif defined(__aarch64__) - -/* Perhaps dmb would be sufficient? Let us be conservative for now. */ -#define mb() { asm volatile("dsb sy" ::: "memory"); } -#define rmb() { asm volatile("dsb ld" ::: "memory"); } -#define wmb() { asm volatile("dsb st" ::: "memory"); } -#define wc_wmb() wmb() - -#else - -#error No architecture specific memory barrier defines found! - -#endif - -#endif /* INFINIBAND_ARCH_H */ diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h deleted file mode 100644 index ea3dade..0000000 --- a/include/infiniband/driver.h +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005 PathScale, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_DRIVER_H -#define INFINIBAND_DRIVER_H - -#include <infiniband/verbs.h> -#include <infiniband/kern-abi.h> - -#ifdef __cplusplus -# define BEGIN_C_DECLS extern "C" { -# define END_C_DECLS } -#else /* !__cplusplus */ -# define BEGIN_C_DECLS -# define END_C_DECLS -#endif /* __cplusplus */ - -/* - * Extension that low-level drivers should add to their .so filename - * (probably via libtool "-release" option). For example a low-level - * driver named "libfoo" should build a plug-in named "libfoo-rdmav2.so". - */ -#define IBV_DEVICE_LIBRARY_EXTENSION rdmav2 - -enum verbs_xrcd_mask { - VERBS_XRCD_HANDLE = 1 << 0, - VERBS_XRCD_RESERVED = 1 << 1 -}; - -struct verbs_xrcd { - struct ibv_xrcd xrcd; - uint32_t comp_mask; - uint32_t handle; -}; - -enum verbs_srq_mask { - VERBS_SRQ_TYPE = 1 << 0, - VERBS_SRQ_XRCD = 1 << 1, - VERBS_SRQ_CQ = 1 << 2, - VERBS_SRQ_NUM = 1 << 3, - VERBS_SRQ_RESERVED = 1 << 4 -}; - -struct verbs_srq { - struct ibv_srq srq; - uint32_t comp_mask; - enum ibv_srq_type srq_type; - struct verbs_xrcd *xrcd; - struct ibv_cq *cq; - uint32_t srq_num; -}; - -enum verbs_qp_mask { - VERBS_QP_XRCD = 1 << 0, - VERBS_QP_RESERVED = 1 << 1 -}; - -enum ibv_gid_type { - IBV_GID_TYPE_IB_ROCE_V1, - IBV_GID_TYPE_ROCE_V2, -}; - -struct verbs_qp { - struct ibv_qp qp; - uint32_t comp_mask; - struct verbs_xrcd *xrcd; -}; -typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path, - int abi_version); -typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path, - int abi_version); - -void ibv_register_driver(const char *name, ibv_driver_init_func init_func); -void verbs_register_driver(const char *name, verbs_driver_init_func init_func); -int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, - size_t cmd_size, struct ibv_get_context_resp *resp, - size_t resp_size); -int ibv_cmd_query_device(struct ibv_context *context, - struct ibv_device_attr *device_attr, - uint64_t *raw_fw_ver, - struct ibv_query_device *cmd, size_t cmd_size); -int ibv_cmd_query_device_ex(struct ibv_context *context, - const struct ibv_query_device_ex_input *input, - struct ibv_device_attr_ex *attr, size_t attr_size, - uint64_t *raw_fw_ver, - struct ibv_query_device_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_query_device_resp_ex *resp, - size_t resp_core_size, - size_t resp_size); -int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr, - struct ibv_query_port *cmd, size_t cmd_size); -int ibv_cmd_query_gid(struct ibv_context *context, uint8_t port_num, - int index, union ibv_gid *gid); -int ibv_cmd_query_pkey(struct ibv_context *context, uint8_t port_num, - int index, uint16_t *pkey); -int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, - struct ibv_alloc_pd *cmd, size_t cmd_size, - struct ibv_alloc_pd_resp *resp, size_t resp_size); -int ibv_cmd_dealloc_pd(struct ibv_pd *pd); -int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd, - int vxrcd_size, - struct ibv_xrcd_init_attr *attr, - struct ibv_open_xrcd *cmd, size_t cmd_size, - struct ibv_open_xrcd_resp *resp, size_t resp_size); -int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd); -#define IBV_CMD_REG_MR_HAS_RESP_PARAMS -int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, int access, - struct ibv_mr *mr, struct ibv_reg_mr *cmd, - size_t cmd_size, - struct ibv_reg_mr_resp *resp, size_t resp_size); -int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, - size_t length, uint64_t hca_va, int access, - struct ibv_pd *pd, struct ibv_rereg_mr *cmd, - size_t cmd_sz, struct ibv_rereg_mr_resp *resp, - size_t resp_sz); -int ibv_cmd_dereg_mr(struct ibv_mr *mr); -int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type, - struct ibv_mw *mw, struct ibv_alloc_mw *cmd, - size_t cmd_size, - struct ibv_alloc_mw_resp *resp, size_t resp_size); -int ibv_cmd_dealloc_mw(struct ibv_mw *mw, - struct ibv_dealloc_mw *cmd, size_t cmd_size); -int ibv_cmd_create_cq(struct ibv_context *context, int cqe, - struct ibv_comp_channel *channel, - int comp_vector, struct ibv_cq *cq, - struct ibv_create_cq *cmd, size_t cmd_size, - struct ibv_create_cq_resp *resp, size_t resp_size); -int ibv_cmd_create_cq_ex(struct ibv_context *context, - struct ibv_cq_init_attr_ex *cq_attr, - struct ibv_cq_ex *cq, - struct ibv_create_cq_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_cq_resp_ex *resp, - size_t resp_core_size, - size_t resp_size); -int ibv_cmd_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); -int ibv_cmd_req_notify_cq(struct ibv_cq *cq, int solicited_only); -#define IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS -int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe, - struct ibv_resize_cq *cmd, size_t cmd_size, - struct ibv_resize_cq_resp *resp, size_t resp_size); -int ibv_cmd_destroy_cq(struct ibv_cq *cq); - -int ibv_cmd_create_srq(struct ibv_pd *pd, - struct ibv_srq *srq, struct ibv_srq_init_attr *attr, - struct ibv_create_srq *cmd, size_t cmd_size, - struct ibv_create_srq_resp *resp, size_t resp_size); -int ibv_cmd_create_srq_ex(struct ibv_context *context, - struct verbs_srq *srq, int vsrq_sz, - struct ibv_srq_init_attr_ex *attr_ex, - struct ibv_create_xsrq *cmd, size_t cmd_size, - struct ibv_create_srq_resp *resp, size_t resp_size); -int ibv_cmd_modify_srq(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask, - struct ibv_modify_srq *cmd, size_t cmd_size); -int ibv_cmd_query_srq(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - struct ibv_query_srq *cmd, size_t cmd_size); -int ibv_cmd_destroy_srq(struct ibv_srq *srq); - -int ibv_cmd_create_qp(struct ibv_pd *pd, - struct ibv_qp *qp, struct ibv_qp_init_attr *attr, - struct ibv_create_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size); -int ibv_cmd_create_qp_ex(struct ibv_context *context, - struct verbs_qp *qp, int vqp_sz, - struct ibv_qp_init_attr_ex *attr_ex, - struct ibv_create_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size); -int ibv_cmd_create_qp_ex2(struct ibv_context *context, - struct verbs_qp *qp, int vqp_sz, - struct ibv_qp_init_attr_ex *qp_attr, - struct ibv_create_qp_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_qp_resp_ex *resp, - size_t resp_core_size, - size_t resp_size); -int ibv_cmd_open_qp(struct ibv_context *context, - struct verbs_qp *qp, int vqp_sz, - struct ibv_qp_open_attr *attr, - struct ibv_open_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size); -int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr, - int attr_mask, - struct ibv_qp_init_attr *qp_init_attr, - struct ibv_query_qp *cmd, size_t cmd_size); -int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_modify_qp *cmd, size_t cmd_size); -int ibv_cmd_destroy_qp(struct ibv_qp *qp); -int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, - struct ibv_send_wr **bad_wr); -int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr); -int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr); -int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, - struct ibv_ah_attr *attr); -int ibv_cmd_destroy_ah(struct ibv_ah *ah); -int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); -int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); - -struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp, - struct ibv_flow_attr *flow_attr); -int ibv_cmd_destroy_flow(struct ibv_flow *flow_id); -int ibv_cmd_create_wq(struct ibv_context *context, - struct ibv_wq_init_attr *wq_init_attr, - struct ibv_wq *wq, - struct ibv_create_wq *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_wq_resp *resp, - size_t resp_core_size, - size_t resp_size); - -int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr, - struct ibv_modify_wq *cmd, size_t cmd_core_size, - size_t cmd_size); -int ibv_cmd_destroy_wq(struct ibv_wq *wq); -int ibv_cmd_create_rwq_ind_table(struct ibv_context *context, - struct ibv_rwq_ind_table_init_attr *init_attr, - struct ibv_rwq_ind_table *rwq_ind_table, - struct ibv_create_rwq_ind_table *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_rwq_ind_table_resp *resp, - size_t resp_core_size, - size_t resp_size); -int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table); -int ibv_dontfork_range(void *base, size_t size); -int ibv_dofork_range(void *base, size_t size); - -/* - * sysfs helper functions - */ -const char *ibv_get_sysfs_path(void); - -int ibv_read_sysfs_file(const char *dir, const char *file, - char *buf, size_t size); - -static inline int verbs_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) -{ - struct verbs_srq *vsrq = container_of(srq, struct verbs_srq, srq); - if (vsrq->comp_mask & VERBS_SRQ_NUM) { - *srq_num = vsrq->srq_num; - return 0; - } - return ENOSYS; -} - -int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, - unsigned int index, enum ibv_gid_type *type); -#endif /* INFINIBAND_DRIVER_H */ diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h deleted file mode 100644 index 381fd50..0000000 --- a/include/infiniband/kern-abi.h +++ /dev/null @@ -1,1252 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * Copyright (c) 2005 PathScale, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef KERN_ABI_H -#define KERN_ABI_H - -#include <linux/types.h> - -/* - * This file must be kept in sync with the kernel's version of - * drivers/infiniband/include/ib_user_verbs.h - */ - -/* - * The minimum and maximum kernel ABI that we can handle. - */ -#define IB_USER_VERBS_MIN_ABI_VERSION 3 -#define IB_USER_VERBS_MAX_ABI_VERSION 6 - -#define IB_USER_VERBS_CMD_THRESHOLD 50 - -enum { - IB_USER_VERBS_CMD_GET_CONTEXT, - IB_USER_VERBS_CMD_QUERY_DEVICE, - IB_USER_VERBS_CMD_QUERY_PORT, - IB_USER_VERBS_CMD_ALLOC_PD, - IB_USER_VERBS_CMD_DEALLOC_PD, - IB_USER_VERBS_CMD_CREATE_AH, - IB_USER_VERBS_CMD_MODIFY_AH, - IB_USER_VERBS_CMD_QUERY_AH, - IB_USER_VERBS_CMD_DESTROY_AH, - IB_USER_VERBS_CMD_REG_MR, - IB_USER_VERBS_CMD_REG_SMR, - IB_USER_VERBS_CMD_REREG_MR, - IB_USER_VERBS_CMD_QUERY_MR, - IB_USER_VERBS_CMD_DEREG_MR, - IB_USER_VERBS_CMD_ALLOC_MW, - IB_USER_VERBS_CMD_BIND_MW, - IB_USER_VERBS_CMD_DEALLOC_MW, - IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL, - IB_USER_VERBS_CMD_CREATE_CQ, - IB_USER_VERBS_CMD_RESIZE_CQ, - IB_USER_VERBS_CMD_DESTROY_CQ, - IB_USER_VERBS_CMD_POLL_CQ, - IB_USER_VERBS_CMD_PEEK_CQ, - IB_USER_VERBS_CMD_REQ_NOTIFY_CQ, - IB_USER_VERBS_CMD_CREATE_QP, - IB_USER_VERBS_CMD_QUERY_QP, - IB_USER_VERBS_CMD_MODIFY_QP, - IB_USER_VERBS_CMD_DESTROY_QP, - IB_USER_VERBS_CMD_POST_SEND, - IB_USER_VERBS_CMD_POST_RECV, - IB_USER_VERBS_CMD_ATTACH_MCAST, - IB_USER_VERBS_CMD_DETACH_MCAST, - IB_USER_VERBS_CMD_CREATE_SRQ, - IB_USER_VERBS_CMD_MODIFY_SRQ, - IB_USER_VERBS_CMD_QUERY_SRQ, - IB_USER_VERBS_CMD_DESTROY_SRQ, - IB_USER_VERBS_CMD_POST_SRQ_RECV, - IB_USER_VERBS_CMD_OPEN_XRCD, - IB_USER_VERBS_CMD_CLOSE_XRCD, - IB_USER_VERBS_CMD_CREATE_XSRQ, - IB_USER_VERBS_CMD_OPEN_QP -}; - -#define IB_USER_VERBS_CMD_COMMAND_MASK 0xff -#define IB_USER_VERBS_CMD_FLAGS_MASK 0xff000000u -#define IB_USER_VERBS_CMD_FLAGS_SHIFT 24 - - -#define IB_USER_VERBS_CMD_FLAG_EXTENDED 0x80ul - -/* use this mask for creating extended commands */ -#define IB_USER_VERBS_CMD_EXTENDED_MASK \ - (IB_USER_VERBS_CMD_FLAG_EXTENDED << \ - IB_USER_VERBS_CMD_FLAGS_SHIFT) - - -enum { - IB_USER_VERBS_CMD_QUERY_DEVICE_EX = IB_USER_VERBS_CMD_EXTENDED_MASK | - IB_USER_VERBS_CMD_QUERY_DEVICE, - IB_USER_VERBS_CMD_CREATE_QP_EX = IB_USER_VERBS_CMD_EXTENDED_MASK | - IB_USER_VERBS_CMD_CREATE_QP, - IB_USER_VERBS_CMD_CREATE_CQ_EX = IB_USER_VERBS_CMD_EXTENDED_MASK | - IB_USER_VERBS_CMD_CREATE_CQ, - IB_USER_VERBS_CMD_CREATE_FLOW = IB_USER_VERBS_CMD_EXTENDED_MASK + - IB_USER_VERBS_CMD_THRESHOLD, - IB_USER_VERBS_CMD_DESTROY_FLOW, - IB_USER_VERBS_CMD_CREATE_WQ, - IB_USER_VERBS_CMD_MODIFY_WQ, - IB_USER_VERBS_CMD_DESTROY_WQ, - IB_USER_VERBS_CMD_CREATE_RWQ_IND_TBL, - IB_USER_VERBS_CMD_DESTROY_RWQ_IND_TBL, -}; - -/* - * Make sure that all structs defined in this file remain laid out so - * that they pack the same way on 32-bit and 64-bit architectures (to - * avoid incompatibility between 32-bit userspace and 64-bit kernels). - * Specifically: - * - Do not use pointer types -- pass pointers in __u64 instead. - * - Make sure that any structure larger than 4 bytes is padded to a - * multiple of 8 bytes. Otherwise the structure size will be - * different between 32-bit and 64-bit architectures. - */ - -struct hdr { - __u32 command; - __u16 in_words; - __u16 out_words; -}; - -struct response_hdr { - __u64 response; -}; - -struct ex_hdr { - struct { - __u32 command; - __u16 in_words; - __u16 out_words; - }; - struct { - __u64 response; - }; - struct { - __u16 provider_in_words; - __u16 provider_out_words; - __u32 reserved; - }; -}; - -struct ibv_kern_async_event { - __u64 element; - __u32 event_type; - __u32 reserved; -}; - -struct ibv_comp_event { - __u64 cq_handle; -}; - -/* - * All commands from userspace should start with a __u32 command field - * followed by __u16 in_words and out_words fields (which give the - * length of the command block and response buffer if any in 32-bit - * words). The kernel driver will read these fields first and read - * the rest of the command struct based on these value. - */ - -struct ibv_query_params { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; -}; - -struct ibv_query_params_resp { - __u32 num_cq_events; -}; - -struct ibv_get_context { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 driver_data[0]; -}; - -struct ibv_get_context_resp { - __u32 async_fd; - __u32 num_comp_vectors; -}; - -struct ibv_query_device { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 driver_data[0]; -}; - -struct ibv_query_device_resp { - __u64 fw_ver; - __u64 node_guid; - __u64 sys_image_guid; - __u64 max_mr_size; - __u64 page_size_cap; - __u32 vendor_id; - __u32 vendor_part_id; - __u32 hw_ver; - __u32 max_qp; - __u32 max_qp_wr; - __u32 device_cap_flags; - __u32 max_sge; - __u32 max_sge_rd; - __u32 max_cq; - __u32 max_cqe; - __u32 max_mr; - __u32 max_pd; - __u32 max_qp_rd_atom; - __u32 max_ee_rd_atom; - __u32 max_res_rd_atom; - __u32 max_qp_init_rd_atom; - __u32 max_ee_init_rd_atom; - __u32 atomic_cap; - __u32 max_ee; - __u32 max_rdd; - __u32 max_mw; - __u32 max_raw_ipv6_qp; - __u32 max_raw_ethy_qp; - __u32 max_mcast_grp; - __u32 max_mcast_qp_attach; - __u32 max_total_mcast_qp_attach; - __u32 max_ah; - __u32 max_fmr; - __u32 max_map_per_fmr; - __u32 max_srq; - __u32 max_srq_wr; - __u32 max_srq_sge; - __u16 max_pkeys; - __u8 local_ca_ack_delay; - __u8 phys_port_cnt; - __u8 reserved[4]; -}; - -struct ibv_query_device_ex { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 reserved; -}; - -struct ibv_odp_caps_resp { - __u64 general_caps; - struct { - __u32 rc_odp_caps; - __u32 uc_odp_caps; - __u32 ud_odp_caps; - } per_transport_caps; - __u32 reserved; -}; - -struct ibv_rss_caps_resp { - __u32 supported_qpts; - __u32 max_rwq_indirection_tables; - __u32 max_rwq_indirection_table_size; - __u32 reserved; -}; - -struct ibv_query_device_resp_ex { - struct ibv_query_device_resp base; - __u32 comp_mask; - __u32 response_length; - struct ibv_odp_caps_resp odp_caps; - __u64 timestamp_mask; - __u64 hca_core_clock; - __u64 device_cap_flags_ex; - struct ibv_rss_caps_resp rss_caps; - __u32 max_wq_type_rq; - __u32 reserved; -}; - -struct ibv_query_port { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u8 port_num; - __u8 reserved[7]; - __u64 driver_data[0]; -}; - -struct ibv_query_port_resp { - __u32 port_cap_flags; - __u32 max_msg_sz; - __u32 bad_pkey_cntr; - __u32 qkey_viol_cntr; - __u32 gid_tbl_len; - __u16 pkey_tbl_len; - __u16 lid; - __u16 sm_lid; - __u8 state; - __u8 max_mtu; - __u8 active_mtu; - __u8 lmc; - __u8 max_vl_num; - __u8 sm_sl; - __u8 subnet_timeout; - __u8 init_type_reply; - __u8 active_width; - __u8 active_speed; - __u8 phys_state; - __u8 link_layer; - __u8 reserved[2]; -}; - -struct ibv_alloc_pd { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 driver_data[0]; -}; - -struct ibv_alloc_pd_resp { - __u32 pd_handle; -}; - -struct ibv_dealloc_pd { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 pd_handle; -}; - -struct ibv_open_xrcd { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 fd; - __u32 oflags; - __u64 driver_data[0]; -}; - -struct ibv_open_xrcd_resp { - __u32 xrcd_handle; -}; - -struct ibv_close_xrcd { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 xrcd_handle; -}; - -struct ibv_reg_mr { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 start; - __u64 length; - __u64 hca_va; - __u32 pd_handle; - __u32 access_flags; - __u64 driver_data[0]; -}; - -struct ibv_reg_mr_resp { - __u32 mr_handle; - __u32 lkey; - __u32 rkey; -}; - -struct ibv_rereg_mr { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 mr_handle; - __u32 flags; - __u64 start; - __u64 length; - __u64 hca_va; - __u32 pd_handle; - __u32 access_flags; - __u64 driver_data[0]; -}; - -struct ibv_rereg_mr_resp { - __u32 lkey; - __u32 rkey; -}; - -struct ibv_dereg_mr { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 mr_handle; -}; - -struct ibv_alloc_mw { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 pd_handle; - __u8 mw_type; - __u8 reserved[3]; -}; - -struct ibv_alloc_mw_resp { - __u32 mw_handle; - __u32 rkey; -}; - -struct ibv_dealloc_mw { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 mw_handle; - __u32 reserved; -}; - -struct ibv_create_comp_channel { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; -}; - -struct ibv_create_comp_channel_resp { - __u32 fd; -}; - -struct ibv_create_cq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 user_handle; - __u32 cqe; - __u32 comp_vector; - __s32 comp_channel; - __u32 reserved; - __u64 driver_data[0]; -}; - -struct ibv_create_cq_resp { - __u32 cq_handle; - __u32 cqe; -}; - -enum ibv_create_cq_ex_kernel_flags { - IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP = 1 << 0, -}; - -struct ibv_create_cq_ex { - struct ex_hdr hdr; - __u64 user_handle; - __u32 cqe; - __u32 comp_vector; - __s32 comp_channel; - __u32 comp_mask; - __u32 flags; - __u32 reserved; -}; - -struct ibv_create_cq_resp_ex { - struct ibv_create_cq_resp base; - __u32 comp_mask; - __u32 response_length; -}; - -struct ibv_kern_wc { - __u64 wr_id; - __u32 status; - __u32 opcode; - __u32 vendor_err; - __u32 byte_len; - __u32 imm_data; - __u32 qp_num; - __u32 src_qp; - __u32 wc_flags; - __u16 pkey_index; - __u16 slid; - __u8 sl; - __u8 dlid_path_bits; - __u8 port_num; - __u8 reserved; -}; - -struct ibv_poll_cq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 cq_handle; - __u32 ne; -}; - -struct ibv_poll_cq_resp { - __u32 count; - __u32 reserved; - struct ibv_kern_wc wc[0]; -}; - -struct ibv_req_notify_cq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 cq_handle; - __u32 solicited; -}; - -struct ibv_resize_cq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 cq_handle; - __u32 cqe; - __u64 driver_data[0]; -}; - -struct ibv_resize_cq_resp { - __u32 cqe; - __u32 reserved; - __u64 driver_data[0]; -}; - -struct ibv_destroy_cq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 cq_handle; - __u32 reserved; -}; - -struct ibv_destroy_cq_resp { - __u32 comp_events_reported; - __u32 async_events_reported; -}; - -struct ibv_kern_global_route { - __u8 dgid[16]; - __u32 flow_label; - __u8 sgid_index; - __u8 hop_limit; - __u8 traffic_class; - __u8 reserved; -}; - -struct ibv_kern_ah_attr { - struct ibv_kern_global_route grh; - __u16 dlid; - __u8 sl; - __u8 src_path_bits; - __u8 static_rate; - __u8 is_global; - __u8 port_num; - __u8 reserved; -}; - -struct ibv_kern_qp_attr { - __u32 qp_attr_mask; - __u32 qp_state; - __u32 cur_qp_state; - __u32 path_mtu; - __u32 path_mig_state; - __u32 qkey; - __u32 rq_psn; - __u32 sq_psn; - __u32 dest_qp_num; - __u32 qp_access_flags; - - struct ibv_kern_ah_attr ah_attr; - struct ibv_kern_ah_attr alt_ah_attr; - - /* ib_qp_cap */ - __u32 max_send_wr; - __u32 max_recv_wr; - __u32 max_send_sge; - __u32 max_recv_sge; - __u32 max_inline_data; - - __u16 pkey_index; - __u16 alt_pkey_index; - __u8 en_sqd_async_notify; - __u8 sq_draining; - __u8 max_rd_atomic; - __u8 max_dest_rd_atomic; - __u8 min_rnr_timer; - __u8 port_num; - __u8 timeout; - __u8 retry_cnt; - __u8 rnr_retry; - __u8 alt_port_num; - __u8 alt_timeout; - __u8 reserved[5]; -}; - -#define IBV_CREATE_QP_COMMON \ - __u64 user_handle; \ - __u32 pd_handle; \ - __u32 send_cq_handle; \ - __u32 recv_cq_handle; \ - __u32 srq_handle; \ - __u32 max_send_wr; \ - __u32 max_recv_wr; \ - __u32 max_send_sge; \ - __u32 max_recv_sge; \ - __u32 max_inline_data; \ - __u8 sq_sig_all; \ - __u8 qp_type; \ - __u8 is_srq; \ - __u8 reserved - -struct ibv_create_qp { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - IBV_CREATE_QP_COMMON; - __u64 driver_data[0]; -}; - -struct ibv_create_qp_common { - IBV_CREATE_QP_COMMON; -}; - -struct ibv_open_qp { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 user_handle; - __u32 pd_handle; - __u32 qpn; - __u8 qp_type; - __u8 reserved[7]; - __u64 driver_data[0]; -}; - -/* also used for open response */ -struct ibv_create_qp_resp { - __u32 qp_handle; - __u32 qpn; - __u32 max_send_wr; - __u32 max_recv_wr; - __u32 max_send_sge; - __u32 max_recv_sge; - __u32 max_inline_data; - __u32 reserved; -}; - -enum ibv_create_qp_ex_kernel_mask { - IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE = 1 << 0, -}; - -struct ibv_create_qp_ex { - struct ex_hdr hdr; - struct ibv_create_qp_common base; - __u32 comp_mask; - __u32 create_flags; - __u32 ind_tbl_handle; - __u32 reserved1; -}; - -struct ibv_create_qp_resp_ex { - struct ibv_create_qp_resp base; - __u32 comp_mask; - __u32 response_length; -}; - -struct ibv_qp_dest { - __u8 dgid[16]; - __u32 flow_label; - __u16 dlid; - __u16 reserved; - __u8 sgid_index; - __u8 hop_limit; - __u8 traffic_class; - __u8 sl; - __u8 src_path_bits; - __u8 static_rate; - __u8 is_global; - __u8 port_num; -}; - -struct ibv_query_qp { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 qp_handle; - __u32 attr_mask; - __u64 driver_data[0]; -}; - -struct ibv_query_qp_resp { - struct ibv_qp_dest dest; - struct ibv_qp_dest alt_dest; - __u32 max_send_wr; - __u32 max_recv_wr; - __u32 max_send_sge; - __u32 max_recv_sge; - __u32 max_inline_data; - __u32 qkey; - __u32 rq_psn; - __u32 sq_psn; - __u32 dest_qp_num; - __u32 qp_access_flags; - __u16 pkey_index; - __u16 alt_pkey_index; - __u8 qp_state; - __u8 cur_qp_state; - __u8 path_mtu; - __u8 path_mig_state; - __u8 sq_draining; - __u8 max_rd_atomic; - __u8 max_dest_rd_atomic; - __u8 min_rnr_timer; - __u8 port_num; - __u8 timeout; - __u8 retry_cnt; - __u8 rnr_retry; - __u8 alt_port_num; - __u8 alt_timeout; - __u8 sq_sig_all; - __u8 reserved[5]; - __u64 driver_data[0]; -}; - -struct ibv_modify_qp { - __u32 command; - __u16 in_words; - __u16 out_words; - struct ibv_qp_dest dest; - struct ibv_qp_dest alt_dest; - __u32 qp_handle; - __u32 attr_mask; - __u32 qkey; - __u32 rq_psn; - __u32 sq_psn; - __u32 dest_qp_num; - __u32 qp_access_flags; - __u16 pkey_index; - __u16 alt_pkey_index; - __u8 qp_state; - __u8 cur_qp_state; - __u8 path_mtu; - __u8 path_mig_state; - __u8 en_sqd_async_notify; - __u8 max_rd_atomic; - __u8 max_dest_rd_atomic; - __u8 min_rnr_timer; - __u8 port_num; - __u8 timeout; - __u8 retry_cnt; - __u8 rnr_retry; - __u8 alt_port_num; - __u8 alt_timeout; - __u8 reserved[2]; - __u64 driver_data[0]; -}; - -struct ibv_destroy_qp { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 qp_handle; - __u32 reserved; -}; - -struct ibv_destroy_qp_resp { - __u32 events_reported; -}; - -struct ibv_kern_send_wr { - __u64 wr_id; - __u32 num_sge; - __u32 opcode; - __u32 send_flags; - __u32 imm_data; - union { - struct { - __u64 remote_addr; - __u32 rkey; - __u32 reserved; - } rdma; - struct { - __u64 remote_addr; - __u64 compare_add; - __u64 swap; - __u32 rkey; - __u32 reserved; - } atomic; - struct { - __u32 ah; - __u32 remote_qpn; - __u32 remote_qkey; - __u32 reserved; - } ud; - } wr; - union { - struct { - __u32 remote_srqn; - } xrc; - } qp_type; -}; - -struct ibv_kern_eth_filter { - __u8 dst_mac[6]; - __u8 src_mac[6]; - __u16 ether_type; - __u16 vlan_tag; -}; - -struct ibv_kern_spec_eth { - __u32 type; - __u16 size; - __u16 reserved; - struct ibv_kern_eth_filter val; - struct ibv_kern_eth_filter mask; -}; - -struct ibv_kern_ipv4_filter { - __u32 src_ip; - __u32 dst_ip; -}; - -struct ibv_kern_spec_ipv4 { - __u32 type; - __u16 size; - __u16 reserved; - struct ibv_kern_ipv4_filter val; - struct ibv_kern_ipv4_filter mask; -}; - -struct ibv_kern_tcp_udp_filter { - __u16 dst_port; - __u16 src_port; -}; - -struct ibv_kern_spec_tcp_udp { - __u32 type; - __u16 size; - __u16 reserved; - struct ibv_kern_tcp_udp_filter val; - struct ibv_kern_tcp_udp_filter mask; -}; - - -struct ibv_kern_spec { - union { - struct { - __u32 type; - __u16 size; - __u16 reserved; - } hdr; - struct ibv_kern_spec_eth eth; - struct ibv_kern_spec_ipv4 ipv4; - struct ibv_kern_spec_tcp_udp tcp_udp; - }; - -}; - -struct ibv_kern_flow_attr { - __u32 type; - __u16 size; - __u16 priority; - __u8 num_of_specs; - __u8 reserved[2]; - __u8 port; - __u32 flags; - /* Following are the optional layers according to user request - * struct ibv_kern_flow_spec_xxx - * struct ibv_kern_flow_spec_yyy - */ -}; - -struct ibv_post_send { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 qp_handle; - __u32 wr_count; - __u32 sge_count; - __u32 wqe_size; - struct ibv_kern_send_wr send_wr[0]; -}; - -struct ibv_post_send_resp { - __u32 bad_wr; -}; - -struct ibv_kern_recv_wr { - __u64 wr_id; - __u32 num_sge; - __u32 reserved; -}; - -struct ibv_post_recv { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 qp_handle; - __u32 wr_count; - __u32 sge_count; - __u32 wqe_size; - struct ibv_kern_recv_wr recv_wr[0]; -}; - -struct ibv_post_recv_resp { - __u32 bad_wr; -}; - -struct ibv_post_srq_recv { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 srq_handle; - __u32 wr_count; - __u32 sge_count; - __u32 wqe_size; - struct ibv_kern_recv_wr recv_wr[0]; -}; - -struct ibv_post_srq_recv_resp { - __u32 bad_wr; -}; - -struct ibv_create_ah { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 user_handle; - __u32 pd_handle; - __u32 reserved; - struct ibv_kern_ah_attr attr; -}; - -struct ibv_create_ah_resp { - __u32 handle; -}; - -struct ibv_destroy_ah { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 ah_handle; -}; - -struct ibv_attach_mcast { - __u32 command; - __u16 in_words; - __u16 out_words; - __u8 gid[16]; - __u32 qp_handle; - __u16 mlid; - __u16 reserved; - __u64 driver_data[0]; -}; - -struct ibv_create_flow { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 qp_handle; - struct ibv_kern_flow_attr flow_attr; -}; - -struct ibv_create_flow_resp { - __u32 comp_mask; - __u32 flow_handle; -}; - -struct ibv_destroy_flow { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 flow_handle; -}; - -struct ibv_detach_mcast { - __u32 command; - __u16 in_words; - __u16 out_words; - __u8 gid[16]; - __u32 qp_handle; - __u16 mlid; - __u16 reserved; - __u64 driver_data[0]; -}; - -struct ibv_create_srq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 user_handle; - __u32 pd_handle; - __u32 max_wr; - __u32 max_sge; - __u32 srq_limit; - __u64 driver_data[0]; -}; - -struct ibv_create_xsrq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u64 user_handle; - __u32 srq_type; - __u32 pd_handle; - __u32 max_wr; - __u32 max_sge; - __u32 srq_limit; - __u32 reserved; - __u32 xrcd_handle; - __u32 cq_handle; - __u64 driver_data[0]; -}; - -struct ibv_create_srq_resp { - __u32 srq_handle; - __u32 max_wr; - __u32 max_sge; - __u32 srqn; -}; - -struct ibv_modify_srq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 srq_handle; - __u32 attr_mask; - __u32 max_wr; - __u32 srq_limit; - __u64 driver_data[0]; -}; - -struct ibv_query_srq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 srq_handle; - __u32 reserved; - __u64 driver_data[0]; -}; - -struct ibv_query_srq_resp { - __u32 max_wr; - __u32 max_sge; - __u32 srq_limit; - __u32 reserved; -}; - -struct ibv_destroy_srq { - __u32 command; - __u16 in_words; - __u16 out_words; - __u64 response; - __u32 srq_handle; - __u32 reserved; -}; - -struct ibv_destroy_srq_resp { - __u32 events_reported; -}; - -/* - * Compatibility with older ABI versions - */ - -enum { - IB_USER_VERBS_CMD_QUERY_PARAMS_V2, - IB_USER_VERBS_CMD_GET_CONTEXT_V2, - IB_USER_VERBS_CMD_QUERY_DEVICE_V2, - IB_USER_VERBS_CMD_QUERY_PORT_V2, - IB_USER_VERBS_CMD_QUERY_GID_V2, - IB_USER_VERBS_CMD_QUERY_PKEY_V2, - IB_USER_VERBS_CMD_ALLOC_PD_V2, - IB_USER_VERBS_CMD_DEALLOC_PD_V2, - IB_USER_VERBS_CMD_CREATE_AH_V2, - IB_USER_VERBS_CMD_MODIFY_AH_V2, - IB_USER_VERBS_CMD_QUERY_AH_V2, - IB_USER_VERBS_CMD_DESTROY_AH_V2, - IB_USER_VERBS_CMD_REG_MR_V2, - IB_USER_VERBS_CMD_REG_SMR_V2, - IB_USER_VERBS_CMD_REREG_MR_V2, - IB_USER_VERBS_CMD_QUERY_MR_V2, - IB_USER_VERBS_CMD_DEREG_MR_V2, - IB_USER_VERBS_CMD_ALLOC_MW_V2, - IB_USER_VERBS_CMD_BIND_MW_V2, - IB_USER_VERBS_CMD_DEALLOC_MW_V2, - IB_USER_VERBS_CMD_CREATE_CQ_V2, - IB_USER_VERBS_CMD_RESIZE_CQ_V2, - IB_USER_VERBS_CMD_DESTROY_CQ_V2, - IB_USER_VERBS_CMD_POLL_CQ_V2, - IB_USER_VERBS_CMD_PEEK_CQ_V2, - IB_USER_VERBS_CMD_REQ_NOTIFY_CQ_V2, - IB_USER_VERBS_CMD_CREATE_QP_V2, - IB_USER_VERBS_CMD_QUERY_QP_V2, - IB_USER_VERBS_CMD_MODIFY_QP_V2, - IB_USER_VERBS_CMD_DESTROY_QP_V2, - IB_USER_VERBS_CMD_POST_SEND_V2, - IB_USER_VERBS_CMD_POST_RECV_V2, - IB_USER_VERBS_CMD_ATTACH_MCAST_V2, - IB_USER_VERBS_CMD_DETACH_MCAST_V2, - IB_USER_VERBS_CMD_CREATE_SRQ_V2, - IB_USER_VERBS_CMD_MODIFY_SRQ_V2, - IB_USER_VERBS_CMD_QUERY_SRQ_V2, - IB_USER_VERBS_CMD_DESTROY_SRQ_V2, - IB_USER_VERBS_CMD_POST_SRQ_RECV_V2, - /* - * Set commands that didn't exist to -1 so our compile-time - * trick opcodes in IBV_INIT_CMD() doesn't break. - */ - IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1, - IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1, - IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1, - IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1, - IB_USER_VERBS_CMD_OPEN_QP_V2 = -1, - IB_USER_VERBS_CMD_CREATE_FLOW_V2 = -1, - IB_USER_VERBS_CMD_DESTROY_FLOW_V2 = -1, - IB_USER_VERBS_CMD_QUERY_DEVICE_EX_V2 = -1, - IB_USER_VERBS_CMD_CREATE_QP_EX_V2 = -1, - IB_USER_VERBS_CMD_CREATE_CQ_EX_V2 = -1, - IB_USER_VERBS_CMD_CREATE_WQ_V2 = -1, - IB_USER_VERBS_CMD_MODIFY_WQ_V2 = -1, - IB_USER_VERBS_CMD_DESTROY_WQ_V2 = -1, - IB_USER_VERBS_CMD_CREATE_RWQ_IND_TBL_V2 = -1, - IB_USER_VERBS_CMD_DESTROY_RWQ_IND_TBL_V2 = -1, -}; - -struct ibv_modify_srq_v3 { - __u32 command; - __u16 in_words; - __u16 out_words; - __u32 srq_handle; - __u32 attr_mask; - __u32 max_wr; - __u32 max_sge; - __u32 srq_limit; - __u32 reserved; - __u64 driver_data[0]; -}; - -struct ibv_create_qp_resp_v3 { - __u32 qp_handle; - __u32 qpn; -}; - -struct ibv_create_qp_resp_v4 { - __u32 qp_handle; - __u32 qpn; - __u32 max_send_wr; - __u32 max_recv_wr; - __u32 max_send_sge; - __u32 max_recv_sge; - __u32 max_inline_data; -}; - -struct ibv_create_srq_resp_v5 { - __u32 srq_handle; -}; - -struct ibv_create_wq { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 wq_type; - __u64 user_handle; - __u32 pd_handle; - __u32 cq_handle; - __u32 max_wr; - __u32 max_sge; -}; - -struct ibv_create_wq_resp { - __u32 comp_mask; - __u32 response_length; - __u32 wq_handle; - __u32 max_wr; - __u32 max_sge; - __u32 wqn; -}; - -struct ibv_destroy_wq { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 wq_handle; -}; - -struct ibv_destroy_wq_resp { - __u32 comp_mask; - __u32 response_length; - __u32 events_reported; - __u32 reserved; -}; - -struct ibv_modify_wq { - struct ex_hdr hdr; - __u32 attr_mask; - __u32 wq_handle; - __u32 wq_state; - __u32 curr_wq_state; -}; - -struct ibv_create_rwq_ind_table { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 log_ind_tbl_size; - /* Following are wq handles based on log_ind_tbl_size, must be 64 bytes aligned. - * __u32 wq_handle1 - * __u32 wq_handle2 - */ -}; - -struct ibv_create_rwq_ind_table_resp { - __u32 comp_mask; - __u32 response_length; - __u32 ind_tbl_handle; - __u32 ind_tbl_num; -}; - -struct ibv_destroy_rwq_ind_table { - struct ex_hdr hdr; - __u32 comp_mask; - __u32 ind_tbl_handle; -}; - -#endif /* KERN_ABI_H */ diff --git a/include/infiniband/marshall.h b/include/infiniband/marshall.h deleted file mode 100644 index 8be76c5..0000000 --- a/include/infiniband/marshall.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_MARSHALL_H -#define INFINIBAND_MARSHALL_H - -#include <infiniband/verbs.h> -#include <infiniband/sa.h> -#include <infiniband/kern-abi.h> -#include <infiniband/sa-kern-abi.h> - -#ifdef __cplusplus -# define BEGIN_C_DECLS extern "C" { -# define END_C_DECLS } -#else /* !__cplusplus */ -# define BEGIN_C_DECLS -# define END_C_DECLS -#endif /* __cplusplus */ - -BEGIN_C_DECLS - -void ibv_copy_qp_attr_from_kern(struct ibv_qp_attr *dst, - struct ibv_kern_qp_attr *src); - -void ibv_copy_ah_attr_from_kern(struct ibv_ah_attr *dst, - struct ibv_kern_ah_attr *src); - -void ibv_copy_path_rec_from_kern(struct ibv_sa_path_rec *dst, - struct ibv_kern_path_rec *src); - -void ibv_copy_path_rec_to_kern(struct ibv_kern_path_rec *dst, - struct ibv_sa_path_rec *src); - -END_C_DECLS - -#endif /* INFINIBAND_MARSHALL_H */ diff --git a/include/infiniband/opcode.h b/include/infiniband/opcode.h deleted file mode 100644 index fd4bc96..0000000 --- a/include/infiniband/opcode.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_OPCODE_H -#define INFINIBAND_OPCODE_H - -/* - * This macro cleans up the definitions of constants for BTH opcodes. - * It is used to define constants such as IBV_OPCODE_UD_SEND_ONLY, - * which becomes IBV_OPCODE_UD + IBV_OPCODE_SEND_ONLY, and this gives - * the correct value. - * - * In short, user code should use the constants defined using the - * macro rather than worrying about adding together other constants. -*/ -#define IBV_OPCODE(transport, op) \ - IBV_OPCODE_ ## transport ## _ ## op = \ - IBV_OPCODE_ ## transport + IBV_OPCODE_ ## op - -enum { - /* transport types -- just used to define real constants */ - IBV_OPCODE_RC = 0x00, - IBV_OPCODE_UC = 0x20, - IBV_OPCODE_RD = 0x40, - IBV_OPCODE_UD = 0x60, - - /* operations -- just used to define real constants */ - IBV_OPCODE_SEND_FIRST = 0x00, - IBV_OPCODE_SEND_MIDDLE = 0x01, - IBV_OPCODE_SEND_LAST = 0x02, - IBV_OPCODE_SEND_LAST_WITH_IMMEDIATE = 0x03, - IBV_OPCODE_SEND_ONLY = 0x04, - IBV_OPCODE_SEND_ONLY_WITH_IMMEDIATE = 0x05, - IBV_OPCODE_RDMA_WRITE_FIRST = 0x06, - IBV_OPCODE_RDMA_WRITE_MIDDLE = 0x07, - IBV_OPCODE_RDMA_WRITE_LAST = 0x08, - IBV_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE = 0x09, - IBV_OPCODE_RDMA_WRITE_ONLY = 0x0a, - IBV_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE = 0x0b, - IBV_OPCODE_RDMA_READ_REQUEST = 0x0c, - IBV_OPCODE_RDMA_READ_RESPONSE_FIRST = 0x0d, - IBV_OPCODE_RDMA_READ_RESPONSE_MIDDLE = 0x0e, - IBV_OPCODE_RDMA_READ_RESPONSE_LAST = 0x0f, - IBV_OPCODE_RDMA_READ_RESPONSE_ONLY = 0x10, - IBV_OPCODE_ACKNOWLEDGE = 0x11, - IBV_OPCODE_ATOMIC_ACKNOWLEDGE = 0x12, - IBV_OPCODE_COMPARE_SWAP = 0x13, - IBV_OPCODE_FETCH_ADD = 0x14, - - /* real constants follow -- see comment about above IBV_OPCODE() - macro for more details */ - - /* RC */ - IBV_OPCODE(RC, SEND_FIRST), - IBV_OPCODE(RC, SEND_MIDDLE), - IBV_OPCODE(RC, SEND_LAST), - IBV_OPCODE(RC, SEND_LAST_WITH_IMMEDIATE), - IBV_OPCODE(RC, SEND_ONLY), - IBV_OPCODE(RC, SEND_ONLY_WITH_IMMEDIATE), - IBV_OPCODE(RC, RDMA_WRITE_FIRST), - IBV_OPCODE(RC, RDMA_WRITE_MIDDLE), - IBV_OPCODE(RC, RDMA_WRITE_LAST), - IBV_OPCODE(RC, RDMA_WRITE_LAST_WITH_IMMEDIATE), - IBV_OPCODE(RC, RDMA_WRITE_ONLY), - IBV_OPCODE(RC, RDMA_WRITE_ONLY_WITH_IMMEDIATE), - IBV_OPCODE(RC, RDMA_READ_REQUEST), - IBV_OPCODE(RC, RDMA_READ_RESPONSE_FIRST), - IBV_OPCODE(RC, RDMA_READ_RESPONSE_MIDDLE), - IBV_OPCODE(RC, RDMA_READ_RESPONSE_LAST), - IBV_OPCODE(RC, RDMA_READ_RESPONSE_ONLY), - IBV_OPCODE(RC, ACKNOWLEDGE), - IBV_OPCODE(RC, ATOMIC_ACKNOWLEDGE), - IBV_OPCODE(RC, COMPARE_SWAP), - IBV_OPCODE(RC, FETCH_ADD), - - /* UC */ - IBV_OPCODE(UC, SEND_FIRST), - IBV_OPCODE(UC, SEND_MIDDLE), - IBV_OPCODE(UC, SEND_LAST), - IBV_OPCODE(UC, SEND_LAST_WITH_IMMEDIATE), - IBV_OPCODE(UC, SEND_ONLY), - IBV_OPCODE(UC, SEND_ONLY_WITH_IMMEDIATE), - IBV_OPCODE(UC, RDMA_WRITE_FIRST), - IBV_OPCODE(UC, RDMA_WRITE_MIDDLE), - IBV_OPCODE(UC, RDMA_WRITE_LAST), - IBV_OPCODE(UC, RDMA_WRITE_LAST_WITH_IMMEDIATE), - IBV_OPCODE(UC, RDMA_WRITE_ONLY), - IBV_OPCODE(UC, RDMA_WRITE_ONLY_WITH_IMMEDIATE), - - /* RD */ - IBV_OPCODE(RD, SEND_FIRST), - IBV_OPCODE(RD, SEND_MIDDLE), - IBV_OPCODE(RD, SEND_LAST), - IBV_OPCODE(RD, SEND_LAST_WITH_IMMEDIATE), - IBV_OPCODE(RD, SEND_ONLY), - IBV_OPCODE(RD, SEND_ONLY_WITH_IMMEDIATE), - IBV_OPCODE(RD, RDMA_WRITE_FIRST), - IBV_OPCODE(RD, RDMA_WRITE_MIDDLE), - IBV_OPCODE(RD, RDMA_WRITE_LAST), - IBV_OPCODE(RD, RDMA_WRITE_LAST_WITH_IMMEDIATE), - IBV_OPCODE(RD, RDMA_WRITE_ONLY), - IBV_OPCODE(RD, RDMA_WRITE_ONLY_WITH_IMMEDIATE), - IBV_OPCODE(RD, RDMA_READ_REQUEST), - IBV_OPCODE(RD, RDMA_READ_RESPONSE_FIRST), - IBV_OPCODE(RD, RDMA_READ_RESPONSE_MIDDLE), - IBV_OPCODE(RD, RDMA_READ_RESPONSE_LAST), - IBV_OPCODE(RD, RDMA_READ_RESPONSE_ONLY), - IBV_OPCODE(RD, ACKNOWLEDGE), - IBV_OPCODE(RD, ATOMIC_ACKNOWLEDGE), - IBV_OPCODE(RD, COMPARE_SWAP), - IBV_OPCODE(RD, FETCH_ADD), - - /* UD */ - IBV_OPCODE(UD, SEND_ONLY), - IBV_OPCODE(UD, SEND_ONLY_WITH_IMMEDIATE) -}; - -#endif /* INFINIBAND_OPCODE_H */ diff --git a/include/infiniband/sa-kern-abi.h b/include/infiniband/sa-kern-abi.h deleted file mode 100644 index 4927d11..0000000 --- a/include/infiniband/sa-kern-abi.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_SA_KERN_ABI_H -#define INFINIBAND_SA_KERN_ABI_H - -#include <linux/types.h> - -/* - * Obsolete, deprecated names. Will be removed in libibverbs 1.1. - */ -#define ib_kern_path_rec ibv_kern_path_rec - -struct ibv_kern_path_rec { - __u8 dgid[16]; - __u8 sgid[16]; - __u16 dlid; - __u16 slid; - __u32 raw_traffic; - __u32 flow_label; - __u32 reversible; - __u32 mtu; - __u16 pkey; - __u8 hop_limit; - __u8 traffic_class; - __u8 numb_path; - __u8 sl; - __u8 mtu_selector; - __u8 rate_selector; - __u8 rate; - __u8 packet_life_time_selector; - __u8 packet_life_time; - __u8 preference; -}; - -#endif /* INFINIBAND_SA_KERN_ABI_H */ diff --git a/include/infiniband/sa.h b/include/infiniband/sa.h deleted file mode 100644 index 1153e94..0000000 --- a/include/infiniband/sa.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_SA_H -#define INFINIBAND_SA_H - -#include <infiniband/verbs.h> - -struct ibv_sa_path_rec { - /* reserved */ - /* reserved */ - union ibv_gid dgid; - union ibv_gid sgid; - uint16_t dlid; - uint16_t slid; - int raw_traffic; - /* reserved */ - uint32_t flow_label; - uint8_t hop_limit; - uint8_t traffic_class; - int reversible; - uint8_t numb_path; - uint16_t pkey; - /* reserved */ - uint8_t sl; - uint8_t mtu_selector; - uint8_t mtu; - uint8_t rate_selector; - uint8_t rate; - uint8_t packet_life_time_selector; - uint8_t packet_life_time; - uint8_t preference; -}; - -struct ibv_sa_mcmember_rec { - union ibv_gid mgid; - union ibv_gid port_gid; - uint32_t qkey; - uint16_t mlid; - uint8_t mtu_selector; - uint8_t mtu; - uint8_t traffic_class; - uint16_t pkey; - uint8_t rate_selector; - uint8_t rate; - uint8_t packet_life_time_selector; - uint8_t packet_life_time; - uint8_t sl; - uint32_t flow_label; - uint8_t hop_limit; - uint8_t scope; - uint8_t join_state; - int proxy_join; -}; - -struct ibv_sa_service_rec { - uint64_t id; - union ibv_gid gid; - uint16_t pkey; - /* uint16_t resv; */ - uint32_t lease; - uint8_t key[16]; - uint8_t name[64]; - uint8_t data8[16]; - uint16_t data16[8]; - uint32_t data32[4]; - uint64_t data64[2]; -}; - -#define IBV_PATH_RECORD_REVERSIBLE 0x80 - -struct ibv_path_record { - uint64_t service_id; - union ibv_gid dgid; - union ibv_gid sgid; - uint16_t dlid; - uint16_t slid; - uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/ - uint8_t tclass; - uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */ - uint16_t pkey; - uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */ - uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */ - uint8_t rate; /* rate selector-7:6 rate-5:0 */ - uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */ - uint8_t preference; - uint8_t reserved[6]; -}; - -#define IBV_PATH_FLAG_GMP (1<<0) -#define IBV_PATH_FLAG_PRIMARY (1<<1) -#define IBV_PATH_FLAG_ALTERNATE (1<<2) -#define IBV_PATH_FLAG_OUTBOUND (1<<3) -#define IBV_PATH_FLAG_INBOUND (1<<4) -#define IBV_PATH_FLAG_INBOUND_REVERSE (1<<5) -#define IBV_PATH_FLAG_BIDIRECTIONAL (IBV_PATH_FLAG_OUTBOUND | \ - IBV_PATH_FLAG_INBOUND_REVERSE) - -struct ibv_path_data { - uint32_t flags; - uint32_t reserved; - struct ibv_path_record path; -}; - -#endif /* INFINIBAND_SA_H */ diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h deleted file mode 100644 index e994c21..0000000 --- a/include/infiniband/verbs.h +++ /dev/null @@ -1,2276 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. - * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005 PathScale, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef INFINIBAND_VERBS_H -#define INFINIBAND_VERBS_H - -#include <stdint.h> -#include <pthread.h> -#include <stddef.h> -#include <errno.h> -#include <string.h> - -#ifdef __cplusplus -# define BEGIN_C_DECLS extern "C" { -# define END_C_DECLS } -#else /* !__cplusplus */ -# define BEGIN_C_DECLS -# define END_C_DECLS -#endif /* __cplusplus */ - -#if __GNUC__ >= 3 -# define __attribute_const __attribute__((const)) -#else -# define __attribute_const -#endif - -BEGIN_C_DECLS - -union ibv_gid { - uint8_t raw[16]; - struct { - uint64_t subnet_prefix; - uint64_t interface_id; - } global; -}; - -#ifndef container_of -/** - * container_of - cast a member of a structure out to the containing structure - * @ptr: the pointer to the member. - * @type: the type of the container struct this is embedded in. - * @member: the name of the member within the struct. - * - */ -#define container_of(ptr, type, member) \ - ((type *) ((uint8_t *)(ptr) - offsetof(type, member))) -#endif - -#define vext_field_avail(type, fld, sz) (offsetof(type, fld) < (sz)) - -static void *__VERBS_ABI_IS_EXTENDED = ((uint8_t *) NULL) - 1; - -enum ibv_node_type { - IBV_NODE_UNKNOWN = -1, - IBV_NODE_CA = 1, - IBV_NODE_SWITCH, - IBV_NODE_ROUTER, - IBV_NODE_RNIC, - IBV_NODE_USNIC, - IBV_NODE_USNIC_UDP, -}; - -enum ibv_transport_type { - IBV_TRANSPORT_UNKNOWN = -1, - IBV_TRANSPORT_IB = 0, - IBV_TRANSPORT_IWARP, - IBV_TRANSPORT_USNIC, - IBV_TRANSPORT_USNIC_UDP, -}; - -enum ibv_device_cap_flags { - IBV_DEVICE_RESIZE_MAX_WR = 1, - IBV_DEVICE_BAD_PKEY_CNTR = 1 << 1, - IBV_DEVICE_BAD_QKEY_CNTR = 1 << 2, - IBV_DEVICE_RAW_MULTI = 1 << 3, - IBV_DEVICE_AUTO_PATH_MIG = 1 << 4, - IBV_DEVICE_CHANGE_PHY_PORT = 1 << 5, - IBV_DEVICE_UD_AV_PORT_ENFORCE = 1 << 6, - IBV_DEVICE_CURR_QP_STATE_MOD = 1 << 7, - IBV_DEVICE_SHUTDOWN_PORT = 1 << 8, - IBV_DEVICE_INIT_TYPE = 1 << 9, - IBV_DEVICE_PORT_ACTIVE_EVENT = 1 << 10, - IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11, - IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12, - IBV_DEVICE_SRQ_RESIZE = 1 << 13, - IBV_DEVICE_N_NOTIFY_CQ = 1 << 14, - IBV_DEVICE_MEM_WINDOW = 1 << 17, - IBV_DEVICE_UD_IP_CSUM = 1 << 18, - IBV_DEVICE_XRC = 1 << 20, - IBV_DEVICE_MEM_MGT_EXTENSIONS = 1 << 21, - IBV_DEVICE_MEM_WINDOW_TYPE_2A = 1 << 23, - IBV_DEVICE_MEM_WINDOW_TYPE_2B = 1 << 24, - IBV_DEVICE_RC_IP_CSUM = 1 << 25, - IBV_DEVICE_RAW_IP_CSUM = 1 << 26, - IBV_DEVICE_MANAGED_FLOW_STEERING = 1 << 29 -}; - -/* - * Can't extended above ibv_device_cap_flags enum as in some systems/compilers - * enum range is limited to 4 bytes. - */ -#define IBV_DEVICE_RAW_SCATTER_FCS (1ULL << 34) - -enum ibv_atomic_cap { - IBV_ATOMIC_NONE, - IBV_ATOMIC_HCA, - IBV_ATOMIC_GLOB -}; - -struct ibv_device_attr { - char fw_ver[64]; - uint64_t node_guid; - uint64_t sys_image_guid; - uint64_t max_mr_size; - uint64_t page_size_cap; - uint32_t vendor_id; - uint32_t vendor_part_id; - uint32_t hw_ver; - int max_qp; - int max_qp_wr; - int device_cap_flags; - int max_sge; - int max_sge_rd; - int max_cq; - int max_cqe; - int max_mr; - int max_pd; - int max_qp_rd_atom; - int max_ee_rd_atom; - int max_res_rd_atom; - int max_qp_init_rd_atom; - int max_ee_init_rd_atom; - enum ibv_atomic_cap atomic_cap; - int max_ee; - int max_rdd; - int max_mw; - int max_raw_ipv6_qp; - int max_raw_ethy_qp; - int max_mcast_grp; - int max_mcast_qp_attach; - int max_total_mcast_qp_attach; - int max_ah; - int max_fmr; - int max_map_per_fmr; - int max_srq; - int max_srq_wr; - int max_srq_sge; - uint16_t max_pkeys; - uint8_t local_ca_ack_delay; - uint8_t phys_port_cnt; -}; - -/* An extensible input struct for possible future extensions of the - * ibv_query_device_ex verb. */ -struct ibv_query_device_ex_input { - uint32_t comp_mask; -}; - -enum ibv_odp_transport_cap_bits { - IBV_ODP_SUPPORT_SEND = 1 << 0, - IBV_ODP_SUPPORT_RECV = 1 << 1, - IBV_ODP_SUPPORT_WRITE = 1 << 2, - IBV_ODP_SUPPORT_READ = 1 << 3, - IBV_ODP_SUPPORT_ATOMIC = 1 << 4, -}; - -struct ibv_odp_caps { - uint64_t general_caps; - struct { - uint32_t rc_odp_caps; - uint32_t uc_odp_caps; - uint32_t ud_odp_caps; - } per_transport_caps; -}; - -enum ibv_odp_general_caps { - IBV_ODP_SUPPORT = 1 << 0, -}; - -struct ibv_tso_caps { - uint32_t max_tso; - uint32_t supported_qpts; -}; - -/* RX Hash function flags */ -enum ibv_rx_hash_function_flags { - IBV_RX_HASH_FUNC_TOEPLITZ = 1 << 0, -}; - -/* - * RX Hash fields enable to set which incoming packet's field should - * participates in RX Hash. Each flag represent certain packet's field, - * when the flag is set the field that is represented by the flag will - * participate in RX Hash calculation. - * Note: *IPV4 and *IPV6 flags can't be enabled together on the same QP - * and *TCP and *UDP flags can't be enabled together on the same QP. -*/ -enum ibv_rx_hash_fields { - IBV_RX_HASH_SRC_IPV4 = 1 << 0, - IBV_RX_HASH_DST_IPV4 = 1 << 1, - IBV_RX_HASH_SRC_IPV6 = 1 << 2, - IBV_RX_HASH_DST_IPV6 = 1 << 3, - IBV_RX_HASH_SRC_PORT_TCP = 1 << 4, - IBV_RX_HASH_DST_PORT_TCP = 1 << 5, - IBV_RX_HASH_SRC_PORT_UDP = 1 << 6, - IBV_RX_HASH_DST_PORT_UDP = 1 << 7 -}; - -struct ibv_rss_caps { - uint32_t supported_qpts; - uint32_t max_rwq_indirection_tables; - uint32_t max_rwq_indirection_table_size; - uint64_t rx_hash_fields_mask; /* enum ibv_rx_hash_fields */ - uint8_t rx_hash_function; /* enum ibv_rx_hash_function_flags */ -}; - -struct ibv_device_attr_ex { - struct ibv_device_attr orig_attr; - uint32_t comp_mask; - struct ibv_odp_caps odp_caps; - uint64_t completion_timestamp_mask; - uint64_t hca_core_clock; - uint64_t device_cap_flags_ex; - struct ibv_tso_caps tso_caps; - struct ibv_rss_caps rss_caps; - uint32_t max_wq_type_rq; -}; - -enum ibv_mtu { - IBV_MTU_256 = 1, - IBV_MTU_512 = 2, - IBV_MTU_1024 = 3, - IBV_MTU_2048 = 4, - IBV_MTU_4096 = 5 -}; - -enum ibv_port_state { - IBV_PORT_NOP = 0, - IBV_PORT_DOWN = 1, - IBV_PORT_INIT = 2, - IBV_PORT_ARMED = 3, - IBV_PORT_ACTIVE = 4, - IBV_PORT_ACTIVE_DEFER = 5 -}; - -enum { - IBV_LINK_LAYER_UNSPECIFIED, - IBV_LINK_LAYER_INFINIBAND, - IBV_LINK_LAYER_ETHERNET, -}; - -enum ibv_port_cap_flags { - IBV_PORT_SM = 1 << 1, - IBV_PORT_NOTICE_SUP = 1 << 2, - IBV_PORT_TRAP_SUP = 1 << 3, - IBV_PORT_OPT_IPD_SUP = 1 << 4, - IBV_PORT_AUTO_MIGR_SUP = 1 << 5, - IBV_PORT_SL_MAP_SUP = 1 << 6, - IBV_PORT_MKEY_NVRAM = 1 << 7, - IBV_PORT_PKEY_NVRAM = 1 << 8, - IBV_PORT_LED_INFO_SUP = 1 << 9, - IBV_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, - IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, - IBV_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, - IBV_PORT_CM_SUP = 1 << 16, - IBV_PORT_SNMP_TUNNEL_SUP = 1 << 17, - IBV_PORT_REINIT_SUP = 1 << 18, - IBV_PORT_DEVICE_MGMT_SUP = 1 << 19, - IBV_PORT_VENDOR_CLASS_SUP = 1 << 20, - IBV_PORT_DR_NOTICE_SUP = 1 << 21, - IBV_PORT_CAP_MASK_NOTICE_SUP = 1 << 22, - IBV_PORT_BOOT_MGMT_SUP = 1 << 23, - IBV_PORT_LINK_LATENCY_SUP = 1 << 24, - IBV_PORT_CLIENT_REG_SUP = 1 << 25, - IBV_PORT_IP_BASED_GIDS = 1 << 26 -}; - -struct ibv_port_attr { - enum ibv_port_state state; - enum ibv_mtu max_mtu; - enum ibv_mtu active_mtu; - int gid_tbl_len; - uint32_t port_cap_flags; - uint32_t max_msg_sz; - uint32_t bad_pkey_cntr; - uint32_t qkey_viol_cntr; - uint16_t pkey_tbl_len; - uint16_t lid; - uint16_t sm_lid; - uint8_t lmc; - uint8_t max_vl_num; - uint8_t sm_sl; - uint8_t subnet_timeout; - uint8_t init_type_reply; - uint8_t active_width; - uint8_t active_speed; - uint8_t phys_state; - uint8_t link_layer; - uint8_t reserved; -}; - -enum ibv_event_type { - IBV_EVENT_CQ_ERR, - IBV_EVENT_QP_FATAL, - IBV_EVENT_QP_REQ_ERR, - IBV_EVENT_QP_ACCESS_ERR, - IBV_EVENT_COMM_EST, - IBV_EVENT_SQ_DRAINED, - IBV_EVENT_PATH_MIG, - IBV_EVENT_PATH_MIG_ERR, - IBV_EVENT_DEVICE_FATAL, - IBV_EVENT_PORT_ACTIVE, - IBV_EVENT_PORT_ERR, - IBV_EVENT_LID_CHANGE, - IBV_EVENT_PKEY_CHANGE, - IBV_EVENT_SM_CHANGE, - IBV_EVENT_SRQ_ERR, - IBV_EVENT_SRQ_LIMIT_REACHED, - IBV_EVENT_QP_LAST_WQE_REACHED, - IBV_EVENT_CLIENT_REREGISTER, - IBV_EVENT_GID_CHANGE, - IBV_EVENT_WQ_FATAL, -}; - -struct ibv_async_event { - union { - struct ibv_cq *cq; - struct ibv_qp *qp; - struct ibv_srq *srq; - struct ibv_wq *wq; - int port_num; - } element; - enum ibv_event_type event_type; -}; - -enum ibv_wc_status { - IBV_WC_SUCCESS, - IBV_WC_LOC_LEN_ERR, - IBV_WC_LOC_QP_OP_ERR, - IBV_WC_LOC_EEC_OP_ERR, - IBV_WC_LOC_PROT_ERR, - IBV_WC_WR_FLUSH_ERR, - IBV_WC_MW_BIND_ERR, - IBV_WC_BAD_RESP_ERR, - IBV_WC_LOC_ACCESS_ERR, - IBV_WC_REM_INV_REQ_ERR, - IBV_WC_REM_ACCESS_ERR, - IBV_WC_REM_OP_ERR, - IBV_WC_RETRY_EXC_ERR, - IBV_WC_RNR_RETRY_EXC_ERR, - IBV_WC_LOC_RDD_VIOL_ERR, - IBV_WC_REM_INV_RD_REQ_ERR, - IBV_WC_REM_ABORT_ERR, - IBV_WC_INV_EECN_ERR, - IBV_WC_INV_EEC_STATE_ERR, - IBV_WC_FATAL_ERR, - IBV_WC_RESP_TIMEOUT_ERR, - IBV_WC_GENERAL_ERR -}; -const char *ibv_wc_status_str(enum ibv_wc_status status); - -enum ibv_wc_opcode { - IBV_WC_SEND, - IBV_WC_RDMA_WRITE, - IBV_WC_RDMA_READ, - IBV_WC_COMP_SWAP, - IBV_WC_FETCH_ADD, - IBV_WC_BIND_MW, - IBV_WC_LOCAL_INV, - IBV_WC_TSO, -/* - * Set value of IBV_WC_RECV so consumers can test if a completion is a - * receive by testing (opcode & IBV_WC_RECV). - */ - IBV_WC_RECV = 1 << 7, - IBV_WC_RECV_RDMA_WITH_IMM -}; - -enum { - IBV_WC_IP_CSUM_OK_SHIFT = 2 -}; - -enum ibv_create_cq_wc_flags { - IBV_WC_EX_WITH_BYTE_LEN = 1 << 0, - IBV_WC_EX_WITH_IMM = 1 << 1, - IBV_WC_EX_WITH_QP_NUM = 1 << 2, - IBV_WC_EX_WITH_SRC_QP = 1 << 3, - IBV_WC_EX_WITH_SLID = 1 << 4, - IBV_WC_EX_WITH_SL = 1 << 5, - IBV_WC_EX_WITH_DLID_PATH_BITS = 1 << 6, - IBV_WC_EX_WITH_COMPLETION_TIMESTAMP = 1 << 7, -}; - -enum { - IBV_WC_STANDARD_FLAGS = IBV_WC_EX_WITH_BYTE_LEN | - IBV_WC_EX_WITH_IMM | - IBV_WC_EX_WITH_QP_NUM | - IBV_WC_EX_WITH_SRC_QP | - IBV_WC_EX_WITH_SLID | - IBV_WC_EX_WITH_SL | - IBV_WC_EX_WITH_DLID_PATH_BITS -}; - -enum { - IBV_CREATE_CQ_SUP_WC_FLAGS = IBV_WC_STANDARD_FLAGS | - IBV_WC_EX_WITH_COMPLETION_TIMESTAMP -}; - -enum ibv_wc_flags { - IBV_WC_GRH = 1 << 0, - IBV_WC_WITH_IMM = 1 << 1, - IBV_WC_IP_CSUM_OK = 1 << IBV_WC_IP_CSUM_OK_SHIFT, - IBV_WC_WITH_INV = 1 << 3 -}; - -struct ibv_wc { - uint64_t wr_id; - enum ibv_wc_status status; - enum ibv_wc_opcode opcode; - uint32_t vendor_err; - uint32_t byte_len; - /* When (wc_flags & IBV_WC_WITH_IMM): Immediate data in network byte order. - * When (wc_flags & IBV_WC_WITH_INV): Stores the invalidated rkey. - */ - uint32_t imm_data; - uint32_t qp_num; - uint32_t src_qp; - int wc_flags; - uint16_t pkey_index; - uint16_t slid; - uint8_t sl; - uint8_t dlid_path_bits; -}; - -enum ibv_access_flags { - IBV_ACCESS_LOCAL_WRITE = 1, - IBV_ACCESS_REMOTE_WRITE = (1<<1), - IBV_ACCESS_REMOTE_READ = (1<<2), - IBV_ACCESS_REMOTE_ATOMIC = (1<<3), - IBV_ACCESS_MW_BIND = (1<<4), - IBV_ACCESS_ZERO_BASED = (1<<5), - IBV_ACCESS_ON_DEMAND = (1<<6), -}; - -struct ibv_mw_bind_info { - struct ibv_mr *mr; - uint64_t addr; - uint64_t length; - int mw_access_flags; /* use ibv_access_flags */ -}; - -struct ibv_pd { - struct ibv_context *context; - uint32_t handle; -}; - -enum ibv_xrcd_init_attr_mask { - IBV_XRCD_INIT_ATTR_FD = 1 << 0, - IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1, - IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2 -}; - -struct ibv_xrcd_init_attr { - uint32_t comp_mask; - int fd; - int oflags; -}; - -struct ibv_xrcd { - struct ibv_context *context; -}; - -enum ibv_rereg_mr_flags { - IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0), - IBV_REREG_MR_CHANGE_PD = (1 << 1), - IBV_REREG_MR_CHANGE_ACCESS = (1 << 2), - IBV_REREG_MR_KEEP_VALID = (1 << 3), - IBV_REREG_MR_FLAGS_SUPPORTED = ((IBV_REREG_MR_KEEP_VALID << 1) - 1) -}; - -struct ibv_mr { - struct ibv_context *context; - struct ibv_pd *pd; - void *addr; - size_t length; - uint32_t handle; - uint32_t lkey; - uint32_t rkey; -}; - -enum ibv_mw_type { - IBV_MW_TYPE_1 = 1, - IBV_MW_TYPE_2 = 2 -}; - -struct ibv_mw { - struct ibv_context *context; - struct ibv_pd *pd; - uint32_t rkey; - uint32_t handle; - enum ibv_mw_type type; -}; - -struct ibv_global_route { - union ibv_gid dgid; - uint32_t flow_label; - uint8_t sgid_index; - uint8_t hop_limit; - uint8_t traffic_class; -}; - -struct ibv_grh { - uint32_t version_tclass_flow; - uint16_t paylen; - uint8_t next_hdr; - uint8_t hop_limit; - union ibv_gid sgid; - union ibv_gid dgid; -}; - -enum ibv_rate { - IBV_RATE_MAX = 0, - IBV_RATE_2_5_GBPS = 2, - IBV_RATE_5_GBPS = 5, - IBV_RATE_10_GBPS = 3, - IBV_RATE_20_GBPS = 6, - IBV_RATE_30_GBPS = 4, - IBV_RATE_40_GBPS = 7, - IBV_RATE_60_GBPS = 8, - IBV_RATE_80_GBPS = 9, - IBV_RATE_120_GBPS = 10, - IBV_RATE_14_GBPS = 11, - IBV_RATE_56_GBPS = 12, - IBV_RATE_112_GBPS = 13, - IBV_RATE_168_GBPS = 14, - IBV_RATE_25_GBPS = 15, - IBV_RATE_100_GBPS = 16, - IBV_RATE_200_GBPS = 17, - IBV_RATE_300_GBPS = 18 -}; - -/** - * ibv_rate_to_mult - Convert the IB rate enum to a multiple of the - * base rate of 2.5 Gbit/sec. For example, IBV_RATE_5_GBPS will be - * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec. - * @rate: rate to convert. - */ -int ibv_rate_to_mult(enum ibv_rate rate) __attribute_const; - -/** - * mult_to_ibv_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate enum. - * @mult: multiple to convert. - */ -enum ibv_rate mult_to_ibv_rate(int mult) __attribute_const; - -/** - * ibv_rate_to_mbps - Convert the IB rate enum to Mbit/sec. - * For example, IBV_RATE_5_GBPS will return the value 5000. - * @rate: rate to convert. - */ -int ibv_rate_to_mbps(enum ibv_rate rate) __attribute_const; - -/** - * mbps_to_ibv_rate - Convert a Mbit/sec value to an IB rate enum. - * @mbps: value to convert. - */ -enum ibv_rate mbps_to_ibv_rate(int mbps) __attribute_const; - -struct ibv_ah_attr { - struct ibv_global_route grh; - uint16_t dlid; - uint8_t sl; - uint8_t src_path_bits; - uint8_t static_rate; - uint8_t is_global; - uint8_t port_num; -}; - -enum ibv_srq_attr_mask { - IBV_SRQ_MAX_WR = 1 << 0, - IBV_SRQ_LIMIT = 1 << 1 -}; - -struct ibv_srq_attr { - uint32_t max_wr; - uint32_t max_sge; - uint32_t srq_limit; -}; - -struct ibv_srq_init_attr { - void *srq_context; - struct ibv_srq_attr attr; -}; - -enum ibv_srq_type { - IBV_SRQT_BASIC, - IBV_SRQT_XRC -}; - -enum ibv_srq_init_attr_mask { - IBV_SRQ_INIT_ATTR_TYPE = 1 << 0, - IBV_SRQ_INIT_ATTR_PD = 1 << 1, - IBV_SRQ_INIT_ATTR_XRCD = 1 << 2, - IBV_SRQ_INIT_ATTR_CQ = 1 << 3, - IBV_SRQ_INIT_ATTR_RESERVED = 1 << 4 -}; - -struct ibv_srq_init_attr_ex { - void *srq_context; - struct ibv_srq_attr attr; - - uint32_t comp_mask; - enum ibv_srq_type srq_type; - struct ibv_pd *pd; - struct ibv_xrcd *xrcd; - struct ibv_cq *cq; -}; - -enum ibv_wq_type { - IBV_WQT_RQ -}; - -enum ibv_wq_init_attr_mask { - IBV_WQ_INIT_ATTR_RESERVED = 1 << 0, -}; - -struct ibv_wq_init_attr { - void *wq_context; - enum ibv_wq_type wq_type; - uint32_t max_wr; - uint32_t max_sge; - struct ibv_pd *pd; - struct ibv_cq *cq; - uint32_t comp_mask; -}; - -enum ibv_wq_state { - IBV_WQS_RESET, - IBV_WQS_RDY, - IBV_WQS_ERR, - IBV_WQS_UNKNOWN -}; - -enum ibv_wq_attr_mask { - IBV_WQ_ATTR_STATE = 1 << 0, - IBV_WQ_ATTR_CURR_STATE = 1 << 1, - IBV_WQ_ATTR_RESERVED = 1 << 2 -}; - -struct ibv_wq_attr { - /* enum ibv_wq_attr_mask */ - uint32_t attr_mask; - /* Move the WQ to this state */ - enum ibv_wq_state wq_state; - /* Assume this is the current WQ state */ - enum ibv_wq_state curr_wq_state; -}; - -/* - * Receive Work Queue Indirection Table. - * It's used in order to distribute incoming packets between different - * Receive Work Queues. Associating Receive WQs with different CPU cores - * allows to workload the traffic between different CPU cores. - * The Indirection Table can contain only WQs of type IBV_WQT_RQ. -*/ -struct ibv_rwq_ind_table { - struct ibv_context *context; - int ind_tbl_handle; - int ind_tbl_num; - uint32_t comp_mask; -}; - -enum ibv_ind_table_init_attr_mask { - IBV_CREATE_IND_TABLE_RESERVED = (1 << 0) -}; - -/* - * Receive Work Queue Indirection Table attributes - */ -struct ibv_rwq_ind_table_init_attr { - uint32_t log_ind_tbl_size; - /* Each entry is a pointer to a Receive Work Queue */ - struct ibv_wq **ind_tbl; - uint32_t comp_mask; -}; - -enum ibv_qp_type { - IBV_QPT_RC = 2, - IBV_QPT_UC, - IBV_QPT_UD, - IBV_QPT_RAW_PACKET = 8, - IBV_QPT_XRC_SEND = 9, - IBV_QPT_XRC_RECV -}; - -struct ibv_qp_cap { - uint32_t max_send_wr; - uint32_t max_recv_wr; - uint32_t max_send_sge; - uint32_t max_recv_sge; - uint32_t max_inline_data; -}; - -struct ibv_qp_init_attr { - void *qp_context; - struct ibv_cq *send_cq; - struct ibv_cq *recv_cq; - struct ibv_srq *srq; - struct ibv_qp_cap cap; - enum ibv_qp_type qp_type; - int sq_sig_all; -}; - -enum ibv_qp_init_attr_mask { - IBV_QP_INIT_ATTR_PD = 1 << 0, - IBV_QP_INIT_ATTR_XRCD = 1 << 1, - IBV_QP_INIT_ATTR_CREATE_FLAGS = 1 << 2, - IBV_QP_INIT_ATTR_MAX_TSO_HEADER = 1 << 3, - IBV_QP_INIT_ATTR_IND_TABLE = 1 << 4, - IBV_QP_INIT_ATTR_RX_HASH = 1 << 5, - IBV_QP_INIT_ATTR_RESERVED = 1 << 6 -}; - -enum ibv_qp_create_flags { - IBV_QP_CREATE_BLOCK_SELF_MCAST_LB = 1 << 1, - IBV_QP_CREATE_SCATTER_FCS = 1 << 8, -}; - -struct ibv_rx_hash_conf { - /* enum ibv_rx_hash_function_flags */ - uint8_t rx_hash_function; - uint8_t rx_hash_key_len; - uint8_t *rx_hash_key; - /* enum ibv_rx_hash_fields */ - uint64_t rx_hash_fields_mask; -}; - -struct ibv_qp_init_attr_ex { - void *qp_context; - struct ibv_cq *send_cq; - struct ibv_cq *recv_cq; - struct ibv_srq *srq; - struct ibv_qp_cap cap; - enum ibv_qp_type qp_type; - int sq_sig_all; - - uint32_t comp_mask; - struct ibv_pd *pd; - struct ibv_xrcd *xrcd; - uint32_t create_flags; - uint16_t max_tso_header; - struct ibv_rwq_ind_table *rwq_ind_tbl; - struct ibv_rx_hash_conf rx_hash_conf; -}; - -enum ibv_qp_open_attr_mask { - IBV_QP_OPEN_ATTR_NUM = 1 << 0, - IBV_QP_OPEN_ATTR_XRCD = 1 << 1, - IBV_QP_OPEN_ATTR_CONTEXT = 1 << 2, - IBV_QP_OPEN_ATTR_TYPE = 1 << 3, - IBV_QP_OPEN_ATTR_RESERVED = 1 << 4 -}; - -struct ibv_qp_open_attr { - uint32_t comp_mask; - uint32_t qp_num; - struct ibv_xrcd *xrcd; - void *qp_context; - enum ibv_qp_type qp_type; -}; - -enum ibv_qp_attr_mask { - IBV_QP_STATE = 1 << 0, - IBV_QP_CUR_STATE = 1 << 1, - IBV_QP_EN_SQD_ASYNC_NOTIFY = 1 << 2, - IBV_QP_ACCESS_FLAGS = 1 << 3, - IBV_QP_PKEY_INDEX = 1 << 4, - IBV_QP_PORT = 1 << 5, - IBV_QP_QKEY = 1 << 6, - IBV_QP_AV = 1 << 7, - IBV_QP_PATH_MTU = 1 << 8, - IBV_QP_TIMEOUT = 1 << 9, - IBV_QP_RETRY_CNT = 1 << 10, - IBV_QP_RNR_RETRY = 1 << 11, - IBV_QP_RQ_PSN = 1 << 12, - IBV_QP_MAX_QP_RD_ATOMIC = 1 << 13, - IBV_QP_ALT_PATH = 1 << 14, - IBV_QP_MIN_RNR_TIMER = 1 << 15, - IBV_QP_SQ_PSN = 1 << 16, - IBV_QP_MAX_DEST_RD_ATOMIC = 1 << 17, - IBV_QP_PATH_MIG_STATE = 1 << 18, - IBV_QP_CAP = 1 << 19, - IBV_QP_DEST_QPN = 1 << 20 -}; - -enum ibv_qp_state { - IBV_QPS_RESET, - IBV_QPS_INIT, - IBV_QPS_RTR, - IBV_QPS_RTS, - IBV_QPS_SQD, - IBV_QPS_SQE, - IBV_QPS_ERR, - IBV_QPS_UNKNOWN -}; - -enum ibv_mig_state { - IBV_MIG_MIGRATED, - IBV_MIG_REARM, - IBV_MIG_ARMED -}; - -struct ibv_qp_attr { - enum ibv_qp_state qp_state; - enum ibv_qp_state cur_qp_state; - enum ibv_mtu path_mtu; - enum ibv_mig_state path_mig_state; - uint32_t qkey; - uint32_t rq_psn; - uint32_t sq_psn; - uint32_t dest_qp_num; - int qp_access_flags; - struct ibv_qp_cap cap; - struct ibv_ah_attr ah_attr; - struct ibv_ah_attr alt_ah_attr; - uint16_t pkey_index; - uint16_t alt_pkey_index; - uint8_t en_sqd_async_notify; - uint8_t sq_draining; - uint8_t max_rd_atomic; - uint8_t max_dest_rd_atomic; - uint8_t min_rnr_timer; - uint8_t port_num; - uint8_t timeout; - uint8_t retry_cnt; - uint8_t rnr_retry; - uint8_t alt_port_num; - uint8_t alt_timeout; -}; - -enum ibv_wr_opcode { - IBV_WR_RDMA_WRITE, - IBV_WR_RDMA_WRITE_WITH_IMM, - IBV_WR_SEND, - IBV_WR_SEND_WITH_IMM, - IBV_WR_RDMA_READ, - IBV_WR_ATOMIC_CMP_AND_SWP, - IBV_WR_ATOMIC_FETCH_AND_ADD, - IBV_WR_LOCAL_INV, - IBV_WR_BIND_MW, - IBV_WR_SEND_WITH_INV, - IBV_WR_TSO, -}; - -enum ibv_send_flags { - IBV_SEND_FENCE = 1 << 0, - IBV_SEND_SIGNALED = 1 << 1, - IBV_SEND_SOLICITED = 1 << 2, - IBV_SEND_INLINE = 1 << 3, - IBV_SEND_IP_CSUM = 1 << 4 -}; - -struct ibv_sge { - uint64_t addr; - uint32_t length; - uint32_t lkey; -}; - -struct ibv_send_wr { - uint64_t wr_id; - struct ibv_send_wr *next; - struct ibv_sge *sg_list; - int num_sge; - enum ibv_wr_opcode opcode; - int send_flags; - uint32_t imm_data; /* in network byte order */ - union { - struct { - uint64_t remote_addr; - uint32_t rkey; - } rdma; - struct { - uint64_t remote_addr; - uint64_t compare_add; - uint64_t swap; - uint32_t rkey; - } atomic; - struct { - struct ibv_ah *ah; - uint32_t remote_qpn; - uint32_t remote_qkey; - } ud; - } wr; - union { - struct { - uint32_t remote_srqn; - } xrc; - } qp_type; - union { - struct { - struct ibv_mw *mw; - uint32_t rkey; - struct ibv_mw_bind_info bind_info; - } bind_mw; - struct { - void *hdr; - uint16_t hdr_sz; - uint16_t mss; - } tso; - }; -}; - -struct ibv_recv_wr { - uint64_t wr_id; - struct ibv_recv_wr *next; - struct ibv_sge *sg_list; - int num_sge; -}; - -struct ibv_mw_bind { - uint64_t wr_id; - int send_flags; - struct ibv_mw_bind_info bind_info; -}; - -struct ibv_srq { - struct ibv_context *context; - void *srq_context; - struct ibv_pd *pd; - uint32_t handle; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t events_completed; -}; - -/* - * Work Queue. QP can be created without internal WQs "packaged" inside it, - * this QP can be configured to use "external" WQ object as its - * receive/send queue. - * WQ associated (many to one) with Completion Queue it owns WQ properties - * (PD, WQ size etc). - * WQ of type IBV_WQT_RQ: - * - Contains receive WQEs, in this case its PD serves as scatter as well. - * - Exposes post receive function to be used to post a list of work - * requests (WRs) to its receive queue. - */ -struct ibv_wq { - struct ibv_context *context; - void *wq_context; - struct ibv_pd *pd; - struct ibv_cq *cq; - uint32_t wq_num; - uint32_t handle; - enum ibv_wq_state state; - enum ibv_wq_type wq_type; - int (*post_recv)(struct ibv_wq *current, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr); - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t events_completed; - uint32_t comp_mask; -}; - -struct ibv_qp { - struct ibv_context *context; - void *qp_context; - struct ibv_pd *pd; - struct ibv_cq *send_cq; - struct ibv_cq *recv_cq; - struct ibv_srq *srq; - uint32_t handle; - uint32_t qp_num; - enum ibv_qp_state state; - enum ibv_qp_type qp_type; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t events_completed; -}; - -struct ibv_comp_channel { - struct ibv_context *context; - int fd; - int refcnt; -}; - -struct ibv_cq { - struct ibv_context *context; - struct ibv_comp_channel *channel; - void *cq_context; - uint32_t handle; - int cqe; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t comp_events_completed; - uint32_t async_events_completed; -}; - -struct ibv_poll_cq_attr { - uint32_t comp_mask; -}; - -struct ibv_cq_ex { - struct ibv_context *context; - struct ibv_comp_channel *channel; - void *cq_context; - uint32_t handle; - int cqe; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t comp_events_completed; - uint32_t async_events_completed; - - uint32_t comp_mask; - enum ibv_wc_status status; - uint64_t wr_id; - int (*start_poll)(struct ibv_cq_ex *current, - struct ibv_poll_cq_attr *attr); - int (*next_poll)(struct ibv_cq_ex *current); - void (*end_poll)(struct ibv_cq_ex *current); - enum ibv_wc_opcode (*read_opcode)(struct ibv_cq_ex *current); - uint32_t (*read_vendor_err)(struct ibv_cq_ex *current); - uint32_t (*read_byte_len)(struct ibv_cq_ex *current); - uint32_t (*read_imm_data)(struct ibv_cq_ex *current); - uint32_t (*read_qp_num)(struct ibv_cq_ex *current); - uint32_t (*read_src_qp)(struct ibv_cq_ex *current); - int (*read_wc_flags)(struct ibv_cq_ex *current); - uint32_t (*read_slid)(struct ibv_cq_ex *current); - uint8_t (*read_sl)(struct ibv_cq_ex *current); - uint8_t (*read_dlid_path_bits)(struct ibv_cq_ex *current); - uint64_t (*read_completion_ts)(struct ibv_cq_ex *current); -}; - -static inline struct ibv_cq *ibv_cq_ex_to_cq(struct ibv_cq_ex *cq) -{ - return (struct ibv_cq *)cq; -} - -static inline int ibv_start_poll(struct ibv_cq_ex *cq, - struct ibv_poll_cq_attr *attr) -{ - return cq->start_poll(cq, attr); -} - -static inline int ibv_next_poll(struct ibv_cq_ex *cq) -{ - return cq->next_poll(cq); -} - -static inline void ibv_end_poll(struct ibv_cq_ex *cq) -{ - cq->end_poll(cq); -} - -static inline enum ibv_wc_opcode ibv_wc_read_opcode(struct ibv_cq_ex *cq) -{ - return cq->read_opcode(cq); -} - -static inline uint32_t ibv_wc_read_vendor_err(struct ibv_cq_ex *cq) -{ - return cq->read_vendor_err(cq); -} - -static inline uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex *cq) -{ - return cq->read_byte_len(cq); -} - -static inline uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex *cq) -{ - return cq->read_imm_data(cq); -} - -static inline uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex *cq) -{ - return cq->read_qp_num(cq); -} - -static inline uint32_t ibv_wc_read_src_qp(struct ibv_cq_ex *cq) -{ - return cq->read_src_qp(cq); -} - -static inline int ibv_wc_read_wc_flags(struct ibv_cq_ex *cq) -{ - return cq->read_wc_flags(cq); -} - -static inline uint32_t ibv_wc_read_slid(struct ibv_cq_ex *cq) -{ - return cq->read_slid(cq); -} - -static inline uint8_t ibv_wc_read_sl(struct ibv_cq_ex *cq) -{ - return cq->read_sl(cq); -} - -static inline uint8_t ibv_wc_read_dlid_path_bits(struct ibv_cq_ex *cq) -{ - return cq->read_dlid_path_bits(cq); -} - -static inline uint64_t ibv_wc_read_completion_ts(struct ibv_cq_ex *cq) -{ - return cq->read_completion_ts(cq); -} - -static inline int ibv_post_wq_recv(struct ibv_wq *wq, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr) -{ - return wq->post_recv(wq, recv_wr, bad_recv_wr); -} - -struct ibv_ah { - struct ibv_context *context; - struct ibv_pd *pd; - uint32_t handle; -}; - -enum ibv_flow_flags { - IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1 << 0, - IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1, -}; - -enum ibv_flow_attr_type { - /* steering according to rule specifications */ - IBV_FLOW_ATTR_NORMAL = 0x0, - /* default unicast and multicast rule - - * receive all Eth traffic which isn't steered to any QP - */ - IBV_FLOW_ATTR_ALL_DEFAULT = 0x1, - /* default multicast rule - - * receive all Eth multicast traffic which isn't steered to any QP - */ - IBV_FLOW_ATTR_MC_DEFAULT = 0x2, -}; - -enum ibv_flow_spec_type { - IBV_FLOW_SPEC_ETH = 0x20, - IBV_FLOW_SPEC_IPV4 = 0x30, - IBV_FLOW_SPEC_TCP = 0x40, - IBV_FLOW_SPEC_UDP = 0x41, -}; - -struct ibv_flow_eth_filter { - uint8_t dst_mac[6]; - uint8_t src_mac[6]; - uint16_t ether_type; - /* - * same layout as 802.1q: prio 3, cfi 1, vlan id 12 - */ - uint16_t vlan_tag; -}; - -struct ibv_flow_spec_eth { - enum ibv_flow_spec_type type; - uint16_t size; - struct ibv_flow_eth_filter val; - struct ibv_flow_eth_filter mask; -}; - -struct ibv_flow_ipv4_filter { - uint32_t src_ip; - uint32_t dst_ip; -}; - -struct ibv_flow_spec_ipv4 { - enum ibv_flow_spec_type type; - uint16_t size; - struct ibv_flow_ipv4_filter val; - struct ibv_flow_ipv4_filter mask; -}; - -struct ibv_flow_tcp_udp_filter { - uint16_t dst_port; - uint16_t src_port; -}; - -struct ibv_flow_spec_tcp_udp { - enum ibv_flow_spec_type type; - uint16_t size; - struct ibv_flow_tcp_udp_filter val; - struct ibv_flow_tcp_udp_filter mask; -}; - -struct ibv_flow_spec { - union { - struct { - enum ibv_flow_spec_type type; - uint16_t size; - } hdr; - struct ibv_flow_spec_eth eth; - struct ibv_flow_spec_ipv4 ipv4; - struct ibv_flow_spec_tcp_udp tcp_udp; - }; -}; - -struct ibv_flow_attr { - uint32_t comp_mask; - enum ibv_flow_attr_type type; - uint16_t size; - uint16_t priority; - uint8_t num_of_specs; - uint8_t port; - uint32_t flags; - /* Following are the optional layers according to user request - * struct ibv_flow_spec_xxx [L2] - * struct ibv_flow_spec_yyy [L3/L4] - */ -}; - -struct ibv_flow { - uint32_t comp_mask; - struct ibv_context *context; - uint32_t handle; -}; - -struct ibv_device; -struct ibv_context; - -struct ibv_device_ops { - struct ibv_context * (*alloc_context)(struct ibv_device *device, int cmd_fd); - void (*free_context)(struct ibv_context *context); -}; - -enum { - IBV_SYSFS_NAME_MAX = 64, - IBV_SYSFS_PATH_MAX = 256 -}; - -struct ibv_device { - struct ibv_device_ops ops; - enum ibv_node_type node_type; - enum ibv_transport_type transport_type; - /* Name of underlying kernel IB device, eg "mthca0" */ - char name[IBV_SYSFS_NAME_MAX]; - /* Name of uverbs device, eg "uverbs0" */ - char dev_name[IBV_SYSFS_NAME_MAX]; - /* Path to infiniband_verbs class device in sysfs */ - char dev_path[IBV_SYSFS_PATH_MAX]; - /* Path to infiniband class device in sysfs */ - char ibdev_path[IBV_SYSFS_PATH_MAX]; -}; - -struct verbs_device { - struct ibv_device device; /* Must be first */ - size_t sz; - size_t size_of_context; - int (*init_context)(struct verbs_device *device, - struct ibv_context *ctx, int cmd_fd); - void (*uninit_context)(struct verbs_device *device, - struct ibv_context *ctx); - /* future fields added here */ -}; - -struct ibv_context_ops { - int (*query_device)(struct ibv_context *context, - struct ibv_device_attr *device_attr); - int (*query_port)(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr); - struct ibv_pd * (*alloc_pd)(struct ibv_context *context); - int (*dealloc_pd)(struct ibv_pd *pd); - struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - int access); - int (*rereg_mr)(struct ibv_mr *mr, - int flags, - struct ibv_pd *pd, void *addr, - size_t length, - int access); - int (*dereg_mr)(struct ibv_mr *mr); - struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type); - int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw, - struct ibv_mw_bind *mw_bind); - int (*dealloc_mw)(struct ibv_mw *mw); - struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, - struct ibv_comp_channel *channel, - int comp_vector); - int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc); - int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only); - void (*cq_event)(struct ibv_cq *cq); - int (*resize_cq)(struct ibv_cq *cq, int cqe); - int (*destroy_cq)(struct ibv_cq *cq); - struct ibv_srq * (*create_srq)(struct ibv_pd *pd, - struct ibv_srq_init_attr *srq_init_attr); - int (*modify_srq)(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask); - int (*query_srq)(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr); - int (*destroy_srq)(struct ibv_srq *srq); - int (*post_srq_recv)(struct ibv_srq *srq, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr); - struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); - int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr); - int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask); - int (*destroy_qp)(struct ibv_qp *qp); - int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr, - struct ibv_send_wr **bad_wr); - int (*post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr); - struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); - int (*destroy_ah)(struct ibv_ah *ah); - int (*attach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, - uint16_t lid); - int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, - uint16_t lid); - void (*async_event)(struct ibv_async_event *event); -}; - -struct ibv_context { - struct ibv_device *device; - struct ibv_context_ops ops; - int cmd_fd; - int async_fd; - int num_comp_vectors; - pthread_mutex_t mutex; - void *abi_compat; -}; - -enum ibv_cq_init_attr_mask { - IBV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 0, - IBV_CQ_INIT_ATTR_MASK_RESERVED = 1 << 1 -}; - -enum ibv_create_cq_attr_flags { - IBV_CREATE_CQ_ATTR_SINGLE_THREADED = 1 << 0, - IBV_CREATE_CQ_ATTR_RESERVED = 1 << 1, -}; - -struct ibv_cq_init_attr_ex { - /* Minimum number of entries required for CQ */ - uint32_t cqe; - /* Consumer-supplied context returned for completion events */ - void *cq_context; - /* Completion channel where completion events will be queued. - * May be NULL if completion events will not be used. - */ - struct ibv_comp_channel *channel; - /* Completion vector used to signal completion events. - * Must be < context->num_comp_vectors. - */ - uint32_t comp_vector; - /* Or'ed bit of enum ibv_create_cq_wc_flags. */ - uint64_t wc_flags; - /* compatibility mask (extended verb). Or'd flags of - * enum ibv_cq_init_attr_mask - */ - uint32_t comp_mask; - /* create cq attr flags - one or more flags from - * enum ibv_create_cq_attr_flags - */ - uint32_t flags; -}; - -enum ibv_values_mask { - IBV_VALUES_MASK_RAW_CLOCK = 1 << 0, - IBV_VALUES_MASK_RESERVED = 1 << 1 -}; - -struct ibv_values_ex { - uint32_t comp_mask; - struct timespec raw_clock; -}; - -enum verbs_context_mask { - VERBS_CONTEXT_XRCD = 1 << 0, - VERBS_CONTEXT_SRQ = 1 << 1, - VERBS_CONTEXT_QP = 1 << 2, - VERBS_CONTEXT_CREATE_FLOW = 1 << 3, - VERBS_CONTEXT_DESTROY_FLOW = 1 << 4, - VERBS_CONTEXT_RESERVED = 1 << 5 -}; - -struct verbs_context { - /* "grows up" - new fields go here */ - int (*destroy_rwq_ind_table)(struct ibv_rwq_ind_table *rwq_ind_table); - struct ibv_rwq_ind_table *(*create_rwq_ind_table)(struct ibv_context *context, - struct ibv_rwq_ind_table_init_attr *init_attr); - int (*destroy_wq)(struct ibv_wq *wq); - int (*modify_wq)(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr); - struct ibv_wq * (*create_wq)(struct ibv_context *context, - struct ibv_wq_init_attr *wq_init_attr); - int (*query_rt_values)(struct ibv_context *context, - struct ibv_values_ex *values); - struct ibv_cq_ex *(*create_cq_ex)(struct ibv_context *context, - struct ibv_cq_init_attr_ex *init_attr); - struct verbs_ex_private *priv; - int (*query_device_ex)(struct ibv_context *context, - const struct ibv_query_device_ex_input *input, - struct ibv_device_attr_ex *attr, - size_t attr_size); - int (*ibv_destroy_flow) (struct ibv_flow *flow); - void (*ABI_placeholder2) (void); /* DO NOT COPY THIS GARBAGE */ - struct ibv_flow * (*ibv_create_flow) (struct ibv_qp *qp, - struct ibv_flow_attr *flow_attr); - void (*ABI_placeholder1) (void); /* DO NOT COPY THIS GARBAGE */ - struct ibv_qp *(*open_qp)(struct ibv_context *context, - struct ibv_qp_open_attr *attr); - struct ibv_qp *(*create_qp_ex)(struct ibv_context *context, - struct ibv_qp_init_attr_ex *qp_init_attr_ex); - int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num); - struct ibv_srq * (*create_srq_ex)(struct ibv_context *context, - struct ibv_srq_init_attr_ex *srq_init_attr_ex); - struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context, - struct ibv_xrcd_init_attr *xrcd_init_attr); - int (*close_xrcd)(struct ibv_xrcd *xrcd); - uint64_t has_comp_mask; - size_t sz; /* Must be immediately before struct ibv_context */ - struct ibv_context context; /* Must be last field in the struct */ -}; - -static inline struct verbs_context *verbs_get_ctx(struct ibv_context *ctx) -{ - return (ctx->abi_compat != __VERBS_ABI_IS_EXTENDED) ? - NULL : container_of(ctx, struct verbs_context, context); -} - -#define verbs_get_ctx_op(ctx, op) ({ \ - struct verbs_context *vctx = verbs_get_ctx(ctx); \ - (!vctx || (vctx->sz < sizeof(*vctx) - offsetof(struct verbs_context, op)) || \ - !vctx->op) ? NULL : vctx; }) - -#define verbs_set_ctx_op(_vctx, op, ptr) ({ \ - struct verbs_context *vctx = _vctx; \ - if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \ - vctx->op = ptr; }) - -static inline struct verbs_device *verbs_get_device( - const struct ibv_device *dev) -{ - return (dev->ops.alloc_context) ? - NULL : container_of(dev, struct verbs_device, device); -} - -/** - * ibv_get_device_list - Get list of IB devices currently available - * @num_devices: optional. if non-NULL, set to the number of devices - * returned in the array. - * - * Return a NULL-terminated array of IB devices. The array can be - * released with ibv_free_device_list(). - */ -struct ibv_device **ibv_get_device_list(int *num_devices); - -/** - * ibv_free_device_list - Free list from ibv_get_device_list() - * - * Free an array of devices returned from ibv_get_device_list(). Once - * the array is freed, pointers to devices that were not opened with - * ibv_open_device() are no longer valid. Client code must open all - * devices it intends to use before calling ibv_free_device_list(). - */ -void ibv_free_device_list(struct ibv_device **list); - -/** - * ibv_get_device_name - Return kernel device name - */ -const char *ibv_get_device_name(struct ibv_device *device); - -/** - * ibv_get_device_guid - Return device's node GUID - */ -uint64_t ibv_get_device_guid(struct ibv_device *device); - -/** - * ibv_open_device - Initialize device for use - */ -struct ibv_context *ibv_open_device(struct ibv_device *device); - -/** - * ibv_close_device - Release device - */ -int ibv_close_device(struct ibv_context *context); - -/** - * ibv_get_async_event - Get next async event - * @event: Pointer to use to return async event - * - * All async events returned by ibv_get_async_event() must eventually - * be acknowledged with ibv_ack_async_event(). - */ -int ibv_get_async_event(struct ibv_context *context, - struct ibv_async_event *event); - -/** - * ibv_ack_async_event - Acknowledge an async event - * @event: Event to be acknowledged. - * - * All async events which are returned by ibv_get_async_event() must - * be acknowledged. To avoid races, destroying an object (CQ, SRQ or - * QP) will wait for all affiliated events to be acknowledged, so - * there should be a one-to-one correspondence between acks and - * successful gets. - */ -void ibv_ack_async_event(struct ibv_async_event *event); - -/** - * ibv_query_device - Get device properties - */ -int ibv_query_device(struct ibv_context *context, - struct ibv_device_attr *device_attr); - -/** - * ibv_query_port - Get port properties - */ -int ibv_query_port(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr); - -static inline int ___ibv_query_port(struct ibv_context *context, - uint8_t port_num, - struct ibv_port_attr *port_attr) -{ - /* For compatibility when running with old libibverbs */ - port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; - port_attr->reserved = 0; - - return ibv_query_port(context, port_num, port_attr); -} - -#define ibv_query_port(context, port_num, port_attr) \ - ___ibv_query_port(context, port_num, port_attr) - -/** - * ibv_query_gid - Get a GID table entry - */ -int ibv_query_gid(struct ibv_context *context, uint8_t port_num, - int index, union ibv_gid *gid); - -/** - * ibv_query_pkey - Get a P_Key table entry - */ -int ibv_query_pkey(struct ibv_context *context, uint8_t port_num, - int index, uint16_t *pkey); - -/** - * ibv_alloc_pd - Allocate a protection domain - */ -struct ibv_pd *ibv_alloc_pd(struct ibv_context *context); - -/** - * ibv_dealloc_pd - Free a protection domain - */ -int ibv_dealloc_pd(struct ibv_pd *pd); - -static inline struct ibv_flow *ibv_create_flow(struct ibv_qp *qp, - struct ibv_flow_attr *flow) -{ - struct verbs_context *vctx = verbs_get_ctx_op(qp->context, - ibv_create_flow); - if (!vctx || !vctx->ibv_create_flow) - return NULL; - - return vctx->ibv_create_flow(qp, flow); -} - -static inline int ibv_destroy_flow(struct ibv_flow *flow_id) -{ - struct verbs_context *vctx = verbs_get_ctx_op(flow_id->context, - ibv_destroy_flow); - if (!vctx || !vctx->ibv_destroy_flow) - return -ENOSYS; - return vctx->ibv_destroy_flow(flow_id); -} - -/** - * ibv_open_xrcd - Open an extended connection domain - */ -static inline struct ibv_xrcd * -ibv_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr) -{ - struct verbs_context *vctx = verbs_get_ctx_op(context, open_xrcd); - if (!vctx) { - errno = ENOSYS; - return NULL; - } - return vctx->open_xrcd(context, xrcd_init_attr); -} - -/** - * ibv_close_xrcd - Close an extended connection domain - */ -static inline int ibv_close_xrcd(struct ibv_xrcd *xrcd) -{ - struct verbs_context *vctx = verbs_get_ctx(xrcd->context); - return vctx->close_xrcd(xrcd); -} - -/** - * ibv_reg_mr - Register a memory region - */ -struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, int access); - - -enum ibv_rereg_mr_err_code { - /* Old MR is valid, invalid input */ - IBV_REREG_MR_ERR_INPUT = -1, - /* Old MR is valid, failed via dont fork on new address range */ - IBV_REREG_MR_ERR_DONT_FORK_NEW = -2, - /* New MR is valid, failed via do fork on old address range */ - IBV_REREG_MR_ERR_DO_FORK_OLD = -3, - /* MR shouldn't be used, command error */ - IBV_REREG_MR_ERR_CMD = -4, - /* MR shouldn't be used, command error, invalid fork state on new address range */ - IBV_REREG_MR_ERR_CMD_AND_DO_FORK_NEW = -5, -}; - -/** - * ibv_rereg_mr - Re-Register a memory region - */ -int ibv_rereg_mr(struct ibv_mr *mr, int flags, - struct ibv_pd *pd, void *addr, - size_t length, int access); -/** - * ibv_dereg_mr - Deregister a memory region - */ -int ibv_dereg_mr(struct ibv_mr *mr); - -/** - * ibv_alloc_mw - Allocate a memory window - */ -static inline struct ibv_mw *ibv_alloc_mw(struct ibv_pd *pd, - enum ibv_mw_type type) -{ - struct ibv_mw *mw; - - if (!pd->context->ops.alloc_mw) { - errno = ENOSYS; - return NULL; - } - - mw = pd->context->ops.alloc_mw(pd, type); - return mw; -} - -/** - * ibv_dealloc_mw - Free a memory window - */ -static inline int ibv_dealloc_mw(struct ibv_mw *mw) -{ - return mw->context->ops.dealloc_mw(mw); -} - -/** - * ibv_inc_rkey - Increase the 8 lsb in the given rkey - */ -static inline uint32_t ibv_inc_rkey(uint32_t rkey) -{ - const uint32_t mask = 0x000000ff; - uint8_t newtag = (uint8_t)((rkey + 1) & mask); - - return (rkey & ~mask) | newtag; -} - -/** - * ibv_bind_mw - Bind a memory window to a region - */ -static inline int ibv_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, - struct ibv_mw_bind *mw_bind) -{ - if (mw->type != IBV_MW_TYPE_1) - return EINVAL; - - return mw->context->ops.bind_mw(qp, mw, mw_bind); -} - -/** - * ibv_create_comp_channel - Create a completion event channel - */ -struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context); - -/** - * ibv_destroy_comp_channel - Destroy a completion event channel - */ -int ibv_destroy_comp_channel(struct ibv_comp_channel *channel); - -/** - * ibv_create_cq - Create a completion queue - * @context - Context CQ will be attached to - * @cqe - Minimum number of entries required for CQ - * @cq_context - Consumer-supplied context returned for completion events - * @channel - Completion channel where completion events will be queued. - * May be NULL if completion events will not be used. - * @comp_vector - Completion vector used to signal completion events. - * Must be >= 0 and < context->num_comp_vectors. - */ -struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, - void *cq_context, - struct ibv_comp_channel *channel, - int comp_vector); - -/** - * ibv_create_cq_ex - Create a completion queue - * @context - Context CQ will be attached to - * @cq_attr - Attributes to create the CQ with - */ -static inline -struct ibv_cq_ex *ibv_create_cq_ex(struct ibv_context *context, - struct ibv_cq_init_attr_ex *cq_attr) -{ - struct verbs_context *vctx = verbs_get_ctx_op(context, create_cq_ex); - - if (!vctx) { - errno = ENOSYS; - return NULL; - } - - if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { - errno = EINVAL; - return NULL; - } - - return vctx->create_cq_ex(context, cq_attr); -} - -/** - * ibv_resize_cq - Modifies the capacity of the CQ. - * @cq: The CQ to resize. - * @cqe: The minimum size of the CQ. - * - * Users can examine the cq structure to determine the actual CQ size. - */ -int ibv_resize_cq(struct ibv_cq *cq, int cqe); - -/** - * ibv_destroy_cq - Destroy a completion queue - */ -int ibv_destroy_cq(struct ibv_cq *cq); - -/** - * ibv_get_cq_event - Read next CQ event - * @channel: Channel to get next event from. - * @cq: Used to return pointer to CQ. - * @cq_context: Used to return consumer-supplied CQ context. - * - * All completion events returned by ibv_get_cq_event() must - * eventually be acknowledged with ibv_ack_cq_events(). - */ -int ibv_get_cq_event(struct ibv_comp_channel *channel, - struct ibv_cq **cq, void **cq_context); - -/** - * ibv_ack_cq_events - Acknowledge CQ completion events - * @cq: CQ to acknowledge events for - * @nevents: Number of events to acknowledge. - * - * All completion events which are returned by ibv_get_cq_event() must - * be acknowledged. To avoid races, ibv_destroy_cq() will wait for - * all completion events to be acknowledged, so there should be a - * one-to-one correspondence between acks and successful gets. An - * application may accumulate multiple completion events and - * acknowledge them in a single call to ibv_ack_cq_events() by passing - * the number of events to ack in @nevents. - */ -void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents); - -/** - * ibv_poll_cq - Poll a CQ for work completions - * @cq:the CQ being polled - * @num_entries:maximum number of completions to return - * @wc:array of at least @num_entries of &struct ibv_wc where completions - * will be returned - * - * Poll a CQ for (possibly multiple) completions. If the return value - * is < 0, an error occurred. If the return value is >= 0, it is the - * number of completions returned. If the return value is - * non-negative and strictly less than num_entries, then the CQ was - * emptied. - */ -static inline int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) -{ - return cq->context->ops.poll_cq(cq, num_entries, wc); -} - -/** - * ibv_req_notify_cq - Request completion notification on a CQ. An - * event will be added to the completion channel associated with the - * CQ when an entry is added to the CQ. - * @cq: The completion queue to request notification for. - * @solicited_only: If non-zero, an event will be generated only for - * the next solicited CQ entry. If zero, any CQ entry, solicited or - * not, will generate an event. - */ -static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only) -{ - return cq->context->ops.req_notify_cq(cq, solicited_only); -} - -/** - * ibv_create_srq - Creates a SRQ associated with the specified protection - * domain. - * @pd: The protection domain associated with the SRQ. - * @srq_init_attr: A list of initial attributes required to create the SRQ. - * - * srq_attr->max_wr and srq_attr->max_sge are read the determine the - * requested size of the SRQ, and set to the actual values allocated - * on return. If ibv_create_srq() succeeds, then max_wr and max_sge - * will always be at least as large as the requested values. - */ -struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, - struct ibv_srq_init_attr *srq_init_attr); - -static inline struct ibv_srq * -ibv_create_srq_ex(struct ibv_context *context, - struct ibv_srq_init_attr_ex *srq_init_attr_ex) -{ - struct verbs_context *vctx; - uint32_t mask = srq_init_attr_ex->comp_mask; - - if (!(mask & ~(IBV_SRQ_INIT_ATTR_PD | IBV_SRQ_INIT_ATTR_TYPE)) && - (mask & IBV_SRQ_INIT_ATTR_PD) && - (!(mask & IBV_SRQ_INIT_ATTR_TYPE) || - (srq_init_attr_ex->srq_type == IBV_SRQT_BASIC))) - return ibv_create_srq(srq_init_attr_ex->pd, - (struct ibv_srq_init_attr *)srq_init_attr_ex); - - vctx = verbs_get_ctx_op(context, create_srq_ex); - if (!vctx) { - errno = ENOSYS; - return NULL; - } - return vctx->create_srq_ex(context, srq_init_attr_ex); -} - -/** - * ibv_modify_srq - Modifies the attributes for the specified SRQ. - * @srq: The SRQ to modify. - * @srq_attr: On input, specifies the SRQ attributes to modify. On output, - * the current values of selected SRQ attributes are returned. - * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ - * are being modified. - * - * The mask may contain IBV_SRQ_MAX_WR to resize the SRQ and/or - * IBV_SRQ_LIMIT to set the SRQ's limit and request notification when - * the number of receives queued drops below the limit. - */ -int ibv_modify_srq(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask); - -/** - * ibv_query_srq - Returns the attribute list and current values for the - * specified SRQ. - * @srq: The SRQ to query. - * @srq_attr: The attributes of the specified SRQ. - */ -int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); - -static inline int ibv_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) -{ - struct verbs_context *vctx = verbs_get_ctx_op(srq->context, get_srq_num); - - if (!vctx) - return ENOSYS; - - return vctx->get_srq_num(srq, srq_num); -} - -/** - * ibv_destroy_srq - Destroys the specified SRQ. - * @srq: The SRQ to destroy. - */ -int ibv_destroy_srq(struct ibv_srq *srq); - -/** - * ibv_post_srq_recv - Posts a list of work requests to the specified SRQ. - * @srq: The SRQ to post the work request on. - * @recv_wr: A list of work requests to post on the receive queue. - * @bad_recv_wr: On an immediate failure, this parameter will reference - * the work request that failed to be posted on the QP. - */ -static inline int ibv_post_srq_recv(struct ibv_srq *srq, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr) -{ - return srq->context->ops.post_srq_recv(srq, recv_wr, bad_recv_wr); -} - -/** - * ibv_create_qp - Create a queue pair. - */ -struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, - struct ibv_qp_init_attr *qp_init_attr); - -static inline struct ibv_qp * -ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_init_attr_ex) -{ - struct verbs_context *vctx; - uint32_t mask = qp_init_attr_ex->comp_mask; - - if (mask == IBV_QP_INIT_ATTR_PD) - return ibv_create_qp(qp_init_attr_ex->pd, - (struct ibv_qp_init_attr *)qp_init_attr_ex); - - vctx = verbs_get_ctx_op(context, create_qp_ex); - if (!vctx) { - errno = ENOSYS; - return NULL; - } - return vctx->create_qp_ex(context, qp_init_attr_ex); -} - -/** - * ibv_query_rt_values_ex - Get current real time @values of a device. - * @values - in/out - defines the attributes we need to query/queried. - * (Or's bits of enum ibv_values_mask on values->comp_mask field) - */ -static inline int -ibv_query_rt_values_ex(struct ibv_context *context, - struct ibv_values_ex *values) -{ - struct verbs_context *vctx; - - vctx = verbs_get_ctx_op(context, query_rt_values); - if (!vctx) - return ENOSYS; - - if (values->comp_mask & ~(IBV_VALUES_MASK_RESERVED - 1)) - return EINVAL; - - return vctx->query_rt_values(context, values); -} - -/** - * ibv_query_device_ex - Get extended device properties - */ -static inline int -ibv_query_device_ex(struct ibv_context *context, - const struct ibv_query_device_ex_input *input, - struct ibv_device_attr_ex *attr) -{ - struct verbs_context *vctx; - int ret; - - vctx = verbs_get_ctx_op(context, query_device_ex); - if (!vctx) - goto legacy; - - ret = vctx->query_device_ex(context, input, attr, sizeof(*attr)); - if (ret == ENOSYS) - goto legacy; - - return ret; - -legacy: - memset(attr, 0, sizeof(*attr)); - ret = ibv_query_device(context, &attr->orig_attr); - - return ret; -} - -/** - * ibv_open_qp - Open a shareable queue pair. - */ -static inline struct ibv_qp * -ibv_open_qp(struct ibv_context *context, struct ibv_qp_open_attr *qp_open_attr) -{ - struct verbs_context *vctx = verbs_get_ctx_op(context, open_qp); - if (!vctx) { - errno = ENOSYS; - return NULL; - } - return vctx->open_qp(context, qp_open_attr); -} - -/** - * ibv_modify_qp - Modify a queue pair. - */ -int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask); - -/** - * ibv_query_qp - Returns the attribute list and current values for the - * specified QP. - * @qp: The QP to query. - * @attr: The attributes of the specified QP. - * @attr_mask: A bit-mask used to select specific attributes to query. - * @init_attr: Additional attributes of the selected QP. - * - * The qp_attr_mask may be used to limit the query to gathering only the - * selected attributes. - */ -int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr); - -/** - * ibv_destroy_qp - Destroy a queue pair. - */ -int ibv_destroy_qp(struct ibv_qp *qp); - -/* - * ibv_create_wq - Creates a WQ associated with the specified protection - * domain. - * @context: ibv_context. - * @wq_init_attr: A list of initial attributes required to create the - * WQ. If WQ creation succeeds, then the attributes are updated to - * the actual capabilities of the created WQ. - * - * wq_init_attr->max_wr and wq_init_attr->max_sge determine - * the requested size of the WQ, and set to the actual values allocated - * on return. - * If ibv_create_wq() succeeds, then max_wr and max_sge will always be - * at least as large as the requested values. - * - * Return Value - * ibv_create_wq() returns a pointer to the created WQ, or NULL if the request - * fails. - */ -static inline struct ibv_wq *ibv_create_wq(struct ibv_context *context, - struct ibv_wq_init_attr *wq_init_attr) -{ - struct verbs_context *vctx = verbs_get_ctx_op(context, create_wq); - struct ibv_wq *wq; - - if (!vctx) { - errno = ENOSYS; - return NULL; - } - - wq = vctx->create_wq(context, wq_init_attr); - if (wq) { - wq->events_completed = 0; - pthread_mutex_init(&wq->mutex, NULL); - pthread_cond_init(&wq->cond, NULL); - } - - return wq; -} - -/* - * ibv_modify_wq - Modifies the attributes for the specified WQ. - * @wq: The WQ to modify. - * @wq_attr: On input, specifies the WQ attributes to modify. - * wq_attr->attr_mask: A bit-mask used to specify which attributes of the WQ - * are being modified. - * On output, the current values of selected WQ attributes are returned. - * - * Return Value - * ibv_modify_wq() returns 0 on success, or the value of errno - * on failure (which indicates the failure reason). - * -*/ -static inline int ibv_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr) -{ - struct verbs_context *vctx = verbs_get_ctx_op(wq->context, modify_wq); - - if (!vctx) - return ENOSYS; - - return vctx->modify_wq(wq, wq_attr); -} - -/* - * ibv_destroy_wq - Destroys the specified WQ. - * @ibv_wq: The WQ to destroy. - * Return Value - * ibv_destroy_wq() returns 0 on success, or the value of errno - * on failure (which indicates the failure reason). -*/ -static inline int ibv_destroy_wq(struct ibv_wq *wq) -{ - struct verbs_context *vctx; - - vctx = verbs_get_ctx_op(wq->context, destroy_wq); - if (!vctx) - return ENOSYS; - - return vctx->destroy_wq(wq); -} - -/* - * ibv_create_rwq_ind_table - Creates a receive work queue Indirection Table - * @context: ibv_context. - * @init_attr: A list of initial attributes required to create the Indirection Table. - * Return Value - * ibv_create_rwq_ind_table returns a pointer to the created - * Indirection Table, or NULL if the request fails. - */ -static inline struct ibv_rwq_ind_table *ibv_create_rwq_ind_table(struct ibv_context *context, - struct ibv_rwq_ind_table_init_attr *init_attr) -{ - struct verbs_context *vctx; - - vctx = verbs_get_ctx_op(context, create_rwq_ind_table); - if (!vctx) { - errno = ENOSYS; - return NULL; - } - - return vctx->create_rwq_ind_table(context, init_attr); -} - -/* - * ibv_destroy_rwq_ind_table - Destroys the specified Indirection Table. - * @rwq_ind_table: The Indirection Table to destroy. - * Return Value - * ibv_destroy_rwq_ind_table() returns 0 on success, or the value of errno - * on failure (which indicates the failure reason). -*/ -static inline int ibv_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) -{ - struct verbs_context *vctx; - - vctx = verbs_get_ctx_op(rwq_ind_table->context, destroy_rwq_ind_table); - if (!vctx) - return ENOSYS; - - return vctx->destroy_rwq_ind_table(rwq_ind_table); -} - -/** - * ibv_post_send - Post a list of work requests to a send queue. - * - * If IBV_SEND_INLINE flag is set, the data buffers can be reused - * immediately after the call returns. - */ -static inline int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, - struct ibv_send_wr **bad_wr) -{ - return qp->context->ops.post_send(qp, wr, bad_wr); -} - -/** - * ibv_post_recv - Post a list of work requests to a receive queue. - */ -static inline int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr) -{ - return qp->context->ops.post_recv(qp, wr, bad_wr); -} - -/** - * ibv_create_ah - Create an address handle. - */ -struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); - -/** - * ibv_init_ah_from_wc - Initializes address handle attributes from a - * work completion. - * @context: Device context on which the received message arrived. - * @port_num: Port on which the received message arrived. - * @wc: Work completion associated with the received message. - * @grh: References the received global route header. This parameter is - * ignored unless the work completion indicates that the GRH is valid. - * @ah_attr: Returned attributes that can be used when creating an address - * handle for replying to the message. - */ -int ibv_init_ah_from_wc(struct ibv_context *context, uint8_t port_num, - struct ibv_wc *wc, struct ibv_grh *grh, - struct ibv_ah_attr *ah_attr); - -/** - * ibv_create_ah_from_wc - Creates an address handle associated with the - * sender of the specified work completion. - * @pd: The protection domain associated with the address handle. - * @wc: Work completion information associated with a received message. - * @grh: References the received global route header. This parameter is - * ignored unless the work completion indicates that the GRH is valid. - * @port_num: The outbound port number to associate with the address. - * - * The address handle is used to reference a local or global destination - * in all UD QP post sends. - */ -struct ibv_ah *ibv_create_ah_from_wc(struct ibv_pd *pd, struct ibv_wc *wc, - struct ibv_grh *grh, uint8_t port_num); - -/** - * ibv_destroy_ah - Destroy an address handle. - */ -int ibv_destroy_ah(struct ibv_ah *ah); - -/** - * ibv_attach_mcast - Attaches the specified QP to a multicast group. - * @qp: QP to attach to the multicast group. The QP must be a UD QP. - * @gid: Multicast group GID. - * @lid: Multicast group LID in host byte order. - * - * In order to route multicast packets correctly, subnet - * administration must have created the multicast group and configured - * the fabric appropriately. The port associated with the specified - * QP must also be a member of the multicast group. - */ -int ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); - -/** - * ibv_detach_mcast - Detaches the specified QP from a multicast group. - * @qp: QP to detach from the multicast group. - * @gid: Multicast group GID. - * @lid: Multicast group LID in host byte order. - */ -int ibv_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); - -/** - * ibv_fork_init - Prepare data structures so that fork() may be used - * safely. If this function is not called or returns a non-zero - * status, then libibverbs data structures are not fork()-safe and the - * effect of an application calling fork() is undefined. - */ -int ibv_fork_init(void); - -/** - * ibv_node_type_str - Return string describing node_type enum value - */ -const char *ibv_node_type_str(enum ibv_node_type node_type); - -/** - * ibv_port_state_str - Return string describing port_state enum value - */ -const char *ibv_port_state_str(enum ibv_port_state port_state); - -/** - * ibv_event_type_str - Return string describing event_type enum value - */ -const char *ibv_event_type_str(enum ibv_event_type event); - -#define ETHERNET_LL_SIZE 6 -int ibv_resolve_eth_l2_from_gid(struct ibv_context *context, - struct ibv_ah_attr *attr, - uint8_t eth_mac[ETHERNET_LL_SIZE], - uint16_t *vid); - -static inline int ibv_is_qpt_supported(uint32_t caps, enum ibv_qp_type qpt) -{ - return !!(caps & (1 << qpt)); -} - -END_C_DECLS - -# undef __attribute_const - - -#endif /* INFINIBAND_VERBS_H */ diff --git a/libibverbs.spec.in b/libibverbs.spec.in deleted file mode 100644 index 040948e..0000000 --- a/libibverbs.spec.in +++ /dev/null @@ -1,169 +0,0 @@ -Name: libibverbs -Version: @VERSION@ -Release: 1%{?dist} -Summary: A library for direct userspace use of RDMA (InfiniBand/iWARP) hardware - -Group: System Environment/Libraries -License: GPLv2 or BSD -Url: http://openfabrics.org/ -Source: http://openfabrics.org/downloads/verbs/libibverbs-@VERSION@.tar.gz -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Requires(post): /sbin/ldconfig -Requires(postun): /sbin/ldconfig - -%description -libibverbs is a library that allows userspace processes to use RDMA -"verbs" as described in the InfiniBand Architecture Specification and -the RDMA Protocol Verbs Specification. This includes direct hardware -access from userspace to InfiniBand/iWARP adapters (kernel bypass) for -fast path operations. - -For this library to be useful, a device-specific plug-in module should -also be installed. - -%package devel -Summary: Development files for the libibverbs library -Group: System Environment/Libraries -Requires: %{name} = %{version}-%{release} - -%description devel -Header files for the libibverbs library. - -%package devel-static -Summary: Static development files for the libibverbs library -Group: System Environment/Libraries - -%description devel-static -Static libraries for the libibverbs library. - -%package utils -Summary: Examples for the libibverbs library -Group: System Environment/Libraries -Requires: %{name} = %{version}-%{release} - -%description utils -Useful libibverbs1 example programs such as ibv_devinfo, which -displays information about RDMA devices. - -%prep -%setup -q -n %{name}-@VERSION@ - -%build -%configure -make %{?_smp_mflags} - -%install -rm -rf $RPM_BUILD_ROOT -make DESTDIR=%{buildroot} install -# remove unpackaged files from the buildroot -rm -f $RPM_BUILD_ROOT%{_libdir}/*.la - -%clean -rm -rf $RPM_BUILD_ROOT - -%post -p /sbin/ldconfig -%postun -p /sbin/ldconfig - -%files -%defattr(-,root,root,-) -%{_libdir}/libibverbs*.so.* -%doc AUTHORS COPYING ChangeLog README - -%files devel -%defattr(-,root,root,-) -%{_libdir}/lib*.so -%{_includedir}/* -%{_mandir}/man3/* - -%files devel-static -%defattr(-,root,root,-) -%{_libdir}/*.a - -%files utils -%defattr(-,root,root,-) -%{_bindir}/* -%{_mandir}/man1/* - -%changelog -* Tue Jul 19 2016 Doug Ledford <dledford@redhat.com> - @VERSION@-1 -- New upstream release - -* Fri Feb 26 2016 Doug Ledford <dledford@redhat.com> - 1.2.0-1 -- New upstream release - -* Tue May 5 2014 Roland Dreier <roland@digitalvampire.org> - 1.1.8-1 -- New upstream release - -* Tue May 28 2013 Roland Dreier <roland@digitalvampire.org> - 1.1.7-1 -- New upstream release - -* Wed Dec 21 2011 Roland Dreier <roland@digitalvampire.org> - 1.1.6-1 -- New upstream release - -* Tue Jun 28 2011 Roland Dreier <roland@digitalvampire.org> - 1.1.5-1 -- New upstream release - -* Thu Jun 3 2010 Roland Dreier <rdreier@cisco.com> - 1.1.4-1 -- New upstream release - -* Thu Oct 29 2009 Roland Dreier <rdreier@cisco.com> - 1.1.3-1 -- New upstream release - -* Fri Jul 24 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.1.2-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild - -* Wed Feb 25 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.1.2-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild - -* Wed Apr 16 2008 Roland Dreier <rdreier@cisco.com> - 1.1.2-1 -- New upstream release -- Update description to mention RDMA and iWARP, not just InfiniBand -- Add "Requires" tag for libibverbs base package to -devel - -* Mon Feb 18 2008 Fedora Release Engineering <rel-eng@fedoraproject.org> - 1.1.1-3 -- Autorebuild for GCC 4.3 - -* Tue Aug 28 2007 Fedora Release Engineering <rel-eng at fedoraproject dot org> - 1.1.1-2 -- Rebuild for selinux ppc32 issue. - -* Fri Jun 15 2007 Roland Dreier <rdreier@cisco.com> - 1.1.1-1 -- New upstream release - -* Wed Apr 11 2007 Roland Dreier <rdreier@cisco.com> - 1.1-1 -- New upstream release - -* Mon May 22 2006 Roland Dreier <rdreier@cisco.com> - 1.1-0.1.rc2 -- New upstream release -- Remove dependency on libsysfs, since it is no longer used -- Put section 3 manpages in devel package. -- Spec file cleanups: remove unused ver macro, improve BuildRoot, add - Requires for /sbin/ldconfig, split static libraries into - devel-static package, and don't use makeinstall any more (all - suggested by Doug Ledford <dledford@redhat.com>). - -* Thu May 4 2006 Roland Dreier <rdreier@cisco.com> - 1.0.4-1 -- New upstream release - -* Mon Mar 14 2006 Roland Dreier <rdreier@cisco.com> - 1.0.3-1 -- New upstream release - -* Mon Mar 13 2006 Roland Dreier <rdreier@cisco.com> - 1.0.1-1 -- New upstream release - -* Thu Feb 16 2006 Roland Dreier <rdreier@cisco.com> - 1.0-1 -- New upstream release - -* Wed Feb 15 2006 Roland Dreier <rolandd@cisco.com> - 1.0-0.5.rc7 -- New upstream release - -* Sun Jan 22 2006 Roland Dreier <rolandd@cisco.com> - 1.0-0.4.rc6 -- New upstream release - -* Tue Oct 25 2005 Roland Dreier <rolandd@cisco.com> - 1.0-0.3.rc5 -- New upstream release - -* Wed Oct 5 2005 Roland Dreier <rolandd@cisco.com> - 1.0-0.2.rc4 -- Update to upstream 1.0-rc4 release - -* Mon Sep 26 2005 Roland Dreier <rolandd@cisco.com> - 1.0-0.1.rc3 -- Initial attempt at Fedora Extras-compliant spec file diff --git a/man/ibv_alloc_mw.3 b/man/ibv_alloc_mw.3 deleted file mode 100644 index 366cefb..0000000 --- a/man/ibv_alloc_mw.3 +++ /dev/null @@ -1,54 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_ALLOC_MW 3 2016-02-02 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_alloc_mw, ibv_dealloc_mw \- allocate or deallocate a memory window (MW) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_mw *ibv_alloc_mw(struct ibv_pd " "*pd" , -.BI " enum ibv_mw_type " "type"); -.sp -.BI "int ibv_dealloc_mw(struct ibv_mw " "*mw" ); -.fi -.SH "DESCRIPTION" -.B ibv_alloc_mw() -allocates a memory window (MW) associated with the protection domain -.I pd\fR. -The MW's type (1 or 2A/2B) is -.I type\fR. -.PP -The MW is created not bound. For it to be useful, the MW must be bound, through either ibv_bind_mw (type 1) or a special WR (type 2). -Once bound, the memory window allows RDMA (remote) access to a subset of the MR to which it was bound, -until invalidated by: ibv_bind_mw verb with zero length for type 1, -IBV_WR_LOCAL_INV/IBV_WR_SEND_WITH_INV WR opcode for type 2, deallocation. -.PP -.B ibv_dealloc_mw() -Unbinds in case was previously bound and deallocates the MW -.I mw\fR. -.SH "RETURN VALUE" -.B ibv_alloc_mw() -returns a pointer to the allocated MW, or NULL if the request fails. -The remote key (\fBR_Key\fR) -field -.B rkey -is used by remote processes to perform Atomic and RDMA operations. This key will be changed during bind operations. The remote process places this -.B rkey -as the rkey field of struct ibv_send_wr passed to the ibv_post_send function. -.PP -.B ibv_dealloc_mw() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_dereg_mr() -fails if any memory window is still bound to this MR. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_post_send (3), -.BR ibv_bind_mw (3), -.BR ibv_reg_mr (3), -.SH "AUTHORS" -.TP -Majd Dibbiny <majd@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_alloc_pd.3 b/man/ibv_alloc_pd.3 deleted file mode 100644 index 63e1aea..0000000 --- a/man/ibv_alloc_pd.3 +++ /dev/null @@ -1,40 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_ALLOC_PD 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_alloc_pd, ibv_dealloc_pd \- allocate or deallocate a protection domain (PDs) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_pd *ibv_alloc_pd(struct ibv_context " "*context" ); -.sp -.BI "int ibv_dealloc_pd(struct ibv_pd " "*pd" ); -.fi -.SH "DESCRIPTION" -.B ibv_alloc_pd() -allocates a PD for the RDMA device context -.I context\fR. -.PP -.B ibv_dealloc_pd() -deallocates the PD -.I pd\fR. -.SH "RETURN VALUE" -.B ibv_alloc_pd() -returns a pointer to the allocated PD, or NULL if the request fails. -.PP -.B ibv_dealloc_pd() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_dealloc_pd() -may fail if any other resource is still associated with the PD being -freed. -.SH "SEE ALSO" -.BR ibv_reg_mr (3), -.BR ibv_create_srq (3), -.BR ibv_create_qp (3), -.BR ibv_create_ah (3), -.BR ibv_create_ah_from_wc (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_asyncwatch.1 b/man/ibv_asyncwatch.1 deleted file mode 100644 index 09f9f65..0000000 --- a/man/ibv_asyncwatch.1 +++ /dev/null @@ -1,30 +0,0 @@ -.TH IBV_ASYNCWATCH 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_asyncwatch \- display asynchronous events - -.SH SYNOPSIS -.B ibv_asyncwatch -[\-d device] [-h] - -.SH DESCRIPTION -.PP -Display asynchronous events forwarded to userspace for an RDMA device. - -.SH OPTIONS - -.PP -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-h\fR, \fB\-\-help\fR=\fIDEVICE\fR -Print a help text and exit. - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > -.TP -Eran Ben Elisha -.RI < eranbe@mellanox.com > diff --git a/man/ibv_attach_mcast.3 b/man/ibv_attach_mcast.3 deleted file mode 100644 index 722ac9f..0000000 --- a/man/ibv_attach_mcast.3 +++ /dev/null @@ -1,53 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_ATTACH_MCAST 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_attach_mcast, ibv_detach_mcast \- attach and detach a queue pair -(QPs) to/from a multicast group -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_attach_mcast(struct ibv_qp " "*qp" ", const union ibv_gid " "*gid" ", -.BI " uint16_t " "lid" "); -.sp -.BI "int ibv_detach_mcast(struct ibv_qp " "*qp" ", const union ibv_gid " "*gid" ", -.BI " uint16_t " "lid" "); -.fi -.SH "DESCRIPTION" -.B ibv_attach_mcast() -attaches the QP -.I qp -to the multicast group having MGID -.I gid -and MLID -.I lid\fR. -.PP -.B ibv_detach_mcast() -detaches the QP -.I qp -to the multicast group having MGID -.I gid -and MLID -.I lid\fR. -.SH "RETURN VALUE" -.B ibv_attach_mcast() -and -.B ibv_detach_mcast() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -Only QPs of Transport Service Type -.BR IBV_QPT_UD -may be attached to multicast groups. -.PP -If a QP is attached to the same multicast group multiple times, the QP will still receive a single copy of a multicast message. -.PP -In order to receive multicast messages, a join request for the -multicast group must be sent to the subnet administrator (SA), so that -the fabric's multicast routing is configured to deliver messages to -the local port. -.SH "SEE ALSO" -.BR ibv_create_qp (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_bind_mw.3 b/man/ibv_bind_mw.3 deleted file mode 100644 index 2943cbc..0000000 --- a/man/ibv_bind_mw.3 +++ /dev/null @@ -1,92 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_BIND_MW 3 2016-02-02 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_bind_mw \- post a request to bind a type 1 memory window to a memory region -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_bind_mw(struct ibv_qp " "*qp" ", struct ibv_mw " "*mw" ", -.BI " struct ibv_mw_bind " "*mw_bind" "); -.fi -.SH "DESCRIPTION" -.B ibv_bind_mw() -posts to the queue pair -.I qp -a request to bind the memory window -.I mw -according to the details in -.I mw_bind\fR. -.PP -The argument -.I mw_bind -is an ibv_mw_bind struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_mw_bind { -.in +8 -uint64_t wr_id; /* User defined WR ID */ -int send_flags; /* Use ibv_send_flags */ -struct ibv_mw_bind_info bind_info; /* MW bind information */ -.in -8 -} -.fi -.PP -.nf -struct ibv_mw_bind_info { -.in +8 -struct ibv_mr *mr; /* The MR to bind the MW to */ -uint64_t addr; /* The address the MW should start at */ -uint64_t length; /* The length (in bytes) the MW should span */ -int mw_access_flags; /* Access flags to the MW. Use ibv_access_flags */ -.in -8 -}; -.fi -.PP -The QP Transport Service Type must be either UC, RC or XRC_SEND for bind operations. -.PP -The attribute send_flags describes the properties of the \s-1WR\s0. It is either 0 or the bitwise \s-1OR\s0 of one or more of the following flags: -.PP -.TP -.B IBV_SEND_FENCE \fR Set the fence indicator. -.TP -.B IBV_SEND_SIGNALED \fR Set the completion notification indicator. Relevant only if QP was created with sq_sig_all=0 -.PP -The mw_access_flags define the allowed access to the MW after the bind -completes successfully. It is either 0 or the bitwise \s-1OR\s0 of one -or more of the following flags: -.TP -.B IBV_ACCESS_REMOTE_WRITE \fR Enable Remote Write Access. Requires local write access to the MR. -.TP -.B IBV_ACCESS_REMOTE_READ\fR Enable Remote Read Access -.TP -.B IBV_ACCESS_REMOTE_ATOMIC\fR Enable Remote Atomic Operation Access (if supported). Requires local write access to the MR. -.TP -.B IBV_ACCESS_ZERO_BASED\fR If set, the address set on the 'remote_addr' field on the WR will be an offset from the MW's start address. -.SH "RETURN VALUE" -.B ibv_bind_mw() -returns 0 on success, or the value of errno on failure (which -indicates the failure reason). In case of a success, the R_key of the -memory window after the bind is returned in the mw_bind->mw->rkey field. -.SH "NOTES" -The bind does not complete when the function return - it is merely -posted to the QP. The user should keep a copy of the old R_key, and -fix the mw structure if the subsequent CQE for the bind operation -indicates a failure. The user may safely send the R_key using a send -request on the same QP, (based on QP ordering rules: a send after a bind -request on the same QP are always ordered), but must not transfer it to the -remote in any other manner before reading a successful CQE. -.PP -Note that for type 2 MW, one should directly post bind WR to the QP, -using ibv_post_send. -.SH "SEE ALSO" -.BR ibv_alloc_mw (3), -.BR ibv_post_send (3), -.BR ibv_poll_cq (3) -.BR ibv_reg_mr (3), -.SH "AUTHORS" -.TP -Majd Dibbiny <majd@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_create_ah.3 b/man/ibv_create_ah.3 deleted file mode 100644 index becc7d1..0000000 --- a/man/ibv_create_ah.3 +++ /dev/null @@ -1,64 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_AH 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_ah, ibv_destroy_ah \- create or destroy an address handle (AH) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_ah *ibv_create_ah(struct ibv_pd " "*pd" ", -.BI " struct ibv_ah_attr " "*attr" "); -.sp -.BI "int ibv_destroy_ah(struct ibv_ah " "*ah" "); -.fi -.SH "DESCRIPTION" -.B ibv_create_ah() -creates an address handle (AH) associated with the protection domain -.I pd\fR. -The argument -.I attr -is an ibv_ah_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_ah_attr { -.in +8 -struct ibv_global_route grh; /* Global Routing Header (GRH) attributes */ -uint16_t dlid; /* Destination LID */ -uint8_t sl; /* Service Level */ -uint8_t src_path_bits; /* Source path bits */ -uint8_t static_rate; /* Maximum static rate */ -uint8_t is_global; /* GRH attributes are valid */ -uint8_t port_num; /* Physical port number */ -.in -8 -}; -.sp -.nf -struct ibv_global_route { -.in +8 -union ibv_gid dgid; /* Destination GID or MGID */ -uint32_t flow_label; /* Flow label */ -uint8_t sgid_index; /* Source GID index */ -uint8_t hop_limit; /* Hop limit */ -uint8_t traffic_class; /* Traffic class */ -.in -8 -}; -.fi -.sp -.PP -.B ibv_destroy_ah() -destroys the AH -.I ah\fR. -.SH "RETURN VALUE" -.B ibv_create_ah() -returns a pointer to the created AH, or NULL if the request fails. -.PP -.B ibv_destroy_ah() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_init_ah_from_wc (3), -.BR ibv_create_ah_from_wc (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_ah_from_wc.3 b/man/ibv_create_ah_from_wc.3 deleted file mode 100644 index 8b92caf..0000000 --- a/man/ibv_create_ah_from_wc.3 +++ /dev/null @@ -1,63 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_AH_FROM_WC 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_init_ah_from_wc, ibv_create_ah_from_wc \- initialize or create an -address handle (AH) from a work completion -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_init_ah_from_wc(struct ibv_context " "*context" ", uint8_t " "port_num" , -.BI " struct ibv_wc " "*wc" ", struct ibv_grh " "*grh" , -.BI " struct ibv_ah_attr " "*ah_attr" ); -.sp -.BI "struct ibv_ah *ibv_create_ah_from_wc(struct ibv_pd " "*pd" , -.BI " struct ibv_wc " "*wc" , -.BI " struct ibv_grh " "*grh" , -.BI " uint8_t " "port_num" ); -.fi -.SH "DESCRIPTION" -.B ibv_init_ah_from_wc() -initializes the address handle (AH) attribute structure -.I ah_attr -for the RDMA device context -.I context -using the port number -.I port_num\fR, -using attributes from the work completion -.I wc -and the Global Routing Header (GRH) structure -.I grh\fR. -.PP -.B ibv_create_ah_from_wc() -creates an AH associated with the protection domain -.I pd -using the port number -.I port_num\fR, -using attributes from the work completion -.I wc -and the Global Routing Header (GRH) structure -.I grh\fR. -.SH "RETURN VALUE" -.B ibv_init_ah_from_wc() -returns 0 on success, and \-1 on error. -.PP -.B ibv_create_ah_from_wc() -returns a pointer to the created AH, or NULL if the request fails. -.SH "NOTES" -The filled structure -.I ah_attr -returned from -.B ibv_init_ah_from_wc() -can be used to create a new AH using -.B ibv_create_ah()\fR. -.SH "SEE ALSO" -.BR ibv_open_device (3), -.BR ibv_alloc_pd (3), -.BR ibv_create_ah (3), -.BR ibv_destroy_ah (3), -.BR ibv_poll_cq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_comp_channel.3 b/man/ibv_create_comp_channel.3 deleted file mode 100644 index 15a9618..0000000 --- a/man/ibv_create_comp_channel.3 +++ /dev/null @@ -1,50 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_COMP_CHANNEL 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_comp_channel, ibv_destroy_comp_channel \- create or -destroy a completion event channel -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context -.BI " " "*context" ); -.sp -.BI "int ibv_destroy_comp_channel(struct ibv_comp_channel " "*channel" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_comp_channel() -creates a completion event channel for the RDMA device context -.I context\fR. -.PP -.B ibv_destroy_comp_channel() -destroys the completion event channel -.I channel\fR. -.SH "RETURN VALUE" -.B ibv_create_comp_channel() -returns a pointer to the created completion event channel, or NULL if the request fails. -.PP -.B ibv_destroy_comp_channel() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -A "completion channel" is an abstraction introduced by libibverbs that -does not exist in the InfiniBand Architecture verbs specification or -RDMA Protocol Verbs Specification. A completion channel is -essentially file descriptor that is used to deliver completion -notifications to a userspace process. When a completion event is -generated for a completion queue (CQ), the event is delivered via the -completion channel attached to that CQ. This may be useful to steer -completion events to different threads by using multiple completion -channels. -.PP -.B ibv_destroy_comp_channel() -fails if any CQs are still associated with the completion event -channel being destroyed. -.SH "SEE ALSO" -.BR ibv_open_device (3), -.BR ibv_create_cq (3), -.BR ibv_get_cq_event (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_cq.3 b/man/ibv_create_cq.3 deleted file mode 100644 index 5dc333e..0000000 --- a/man/ibv_create_cq.3 +++ /dev/null @@ -1,58 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_CQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_cq, ibv_destroy_cq \- create or destroy a completion queue (CQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_cq *ibv_create_cq(struct ibv_context " "*context" ", int " "cqe" , -.BI " void " "*cq_context" , -.BI " struct ibv_comp_channel " "*channel" , -.BI " int " "comp_vector" ); -.sp -.BI "int ibv_destroy_cq(struct ibv_cq " "*cq" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_cq() -creates a completion queue (CQ) with at least -.I cqe -entries for the RDMA device context -.I context\fR. -The pointer -.I cq_context -will be used to set user context pointer of the CQ structure. The argument -.I channel -is optional; if not NULL, the completion channel -.I channel -will be used to return completion events. The CQ will use the -completion vector -.I comp_vector -for signaling completion events; it must be at least zero and less than -.I context\fR->num_comp_vectors. -.PP -.B ibv_destroy_cq() -destroys the CQ -.I cq\fR. -.SH "RETURN VALUE" -.B ibv_create_cq() -returns a pointer to the CQ, or NULL if the request fails. -.PP -.B ibv_destroy_cq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_create_cq() -may create a CQ with size greater than or equal to the requested -size. Check the cqe attribute in the returned CQ for the actual size. -.PP -.B ibv_destroy_cq() -fails if any queue pair is still associated with this CQ. -.SH "SEE ALSO" -.BR ibv_resize_cq (3), -.BR ibv_req_notify_cq (3), -.BR ibv_ack_cq_events (3), -.BR ibv_create_qp (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_cq_ex.3 b/man/ibv_create_cq_ex.3 deleted file mode 100644 index ecaeb0a..0000000 --- a/man/ibv_create_cq_ex.3 +++ /dev/null @@ -1,149 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_CQ_EX 3 2016-05-08 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_cq_ex \- create a completion queue (CQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_cq_ex *ibv_create_cq_ex(struct ibv_context " "*context" ", -.BI " struct ibv_create_cq_attr_ex " "*cq_attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_cq_ex() -creates a completion queue (CQ) for RDMA device context -.I context\fR. -The argument -.I cq_attr -is a pointer to struct ibv_create_cq_attr_ex as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_create_cq_attr_ex { -.in +8 -int cqe; /* Minimum number of entries required for CQ */ -void *cq_context; /* Consumer-supplied context returned for completion events */ -struct ibv_comp_channel *channel; /* Completion channel where completion events will be queued. May be NULL if completion events will not be used. */ -int comp_vector; /* Completion vector used to signal completion events. Must be >= 0 and < context->num_comp_vectors. */ -uint64_t wc_flags; /* The wc_flags that should be returned in ibv_poll_cq_ex. Or'ed bit of enum ibv_wc_flags_ex. */ -uint32_t comp_mask; /* compatibility mask (extended verb). */ -.in -8 -}; - -enum ibv_wc_flags_ex { - IBV_WC_EX_WITH_BYTE_LEN = 1 << 0, /* Require byte len in WC */ - IBV_WC_EX_WITH_IMM = 1 << 1, /* Require immediate in WC */ - IBV_WC_EX_WITH_QP_NUM = 1 << 2, /* Require QP number in WC */ - IBV_WC_EX_WITH_SRC_QP = 1 << 3, /* Require source QP in WC */ - IBV_WC_EX_WITH_SLID = 1 << 4, /* Require slid in WC */ - IBV_WC_EX_WITH_SL = 1 << 5, /* Require sl in WC */ - IBV_WC_EX_WITH_DLID_PATH_BITS = 1 << 6, /* Require dlid path bits in WC */ - IBV_WC_EX_WITH_COMPLETION_TIMESTAMP = 1 << 7, /* Require completion timestamp in WC /* -}; - -enum ibv_cq_init_attr_mask { - IBV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 0, -}; - -enum ibv_create_cq_attr_flags { - IBV_CREATE_CQ_ATTR_SINGLE_THREADED = 1 << 0, /* This CQ is used from a single threaded, thus no locking is required */ -}; - -.SH "Polling an extended CQ" -In order to poll an extended CQ efficiently, a user could use the following functions. - -.TP -.B Completion iterator functions - -.BI "int ibv_start_poll(struct ibv_cq_ex " "*cq" ", struct ibv_poll_cq_attr " "*attr") -.br -Start polling a batch of work completions. -.I attr -is given in order to make this function -easily extensible in the future. This function either returns 0 on success or an error code -otherwise. When no completions are available on the CQ, ENOENT is returned, but the CQ remains -in a valid state. On success, querying the completion's attribute could be done using the query -functions described below. If an error code is given, end_poll shouldn't be called. - -.BI "int ibv_next_poll(struct ibv_cq_ex " "*cq") -.br -This function is called in order to get the next work completion. It has to be called after -.I start_poll -and before -.I end_poll -are called. This function either returns 0 on success or an error code -otherwise. When no completions are available on the CQ, ENOENT is returned, but the CQ remains -in a valid state. On success, querying the completion's attribute could be done using the query -functions described below. If an error code is given, end_poll should still be called, -indicating this is the end of the polled batch. - -.BI "void ibv_end_poll(struct ibv_cq_ex " "*cq") -.br -This function indicates the end of polling batch of work completions. After calling this function, the user should start a new batch -by calling -.I start_poll. - -.TP -.B Polling fields in the completion -Below members and functions are used in order to poll the current completion. The current completion is the completion which the iterator points to (start_poll and next_poll advances this iterator). Only fields that the user requested via wc_flags in ibv_create_cq_ex could be queried. In addition, some fields are only valid in certain opcodes and status codes. - -.BI "uint64_t wr_id - Can be accessed directly from struct ibv_cq_ex". - -.BI "enum ibv_wc_status - Can be accessed directly from struct ibv_cq_ex". - -.BI "enum ibv_wc_opcode ibv_wc_read_opcode(struct ibv_cq_ex " "*cq"); \c - Get the opcode from the current completion. - -.BI "uint32_t ibv_wc_read_vendor_err(struct ibv_cq_ex " "*cq"); \c - Get the vendor error from the current completion. - -.BI "uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex " "*cq"); \c - Get the vendor error from the current completion. - -.BI "uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex " "*cq"); \c - Get the immediate data field from the current completion. - -.BI "uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex " "*cq"); \c - Get the QP number field from the current completion. - -.BI "uint32_t ibv_wc_read_src_qp(struct ibv_cq_ex " "*cq"); \c - Get the source QP number field from the current completion. - -.BI "int ibv_wc_read_wc_flags(struct ibv_cq_ex " "*cq"); \c - Get the QP flags field from the current completion. - -.BI "uint16_t ibv_wc_read_pkey_index(struct ibv_cq_ex " "*cq"); \c - Get the pkey index field from the current completion. - -.BI "uint32_t ibv_wc_read_slid(struct ibv_cq_ex " "*cq"); \c - Get the slid field from the current completion. - -.BI "uint8_t ibv_wc_read_sl(struct ibv_cq_ex " "*cq"); \c - Get the sl field from the current completion. - -.BI "uint8_t ibv_wc_read_dlid_path_bits(struct ibv_cq_ex " "*cq"); \c - Get the dlid_path_bits field from the current completion. - -.BI "uint64_t ibv_wc_read_completion_ts(struct ibv_cq_ex " "*cq"); \c - Get the completion timestamp from the current completion. - -.SH "RETURN VALUE" -.B ibv_create_cq_ex() -returns a pointer to the CQ, or NULL if the request fails. -.SH "NOTES" -.B ibv_create_cq_ex() -may create a CQ with size greater than or equal to the requested -size. Check the cqe attribute in the returned CQ for the actual size. -.PP -CQ should be destroyed with ibv_destroy_cq. -.PP -.SH "SEE ALSO" -.BR ibv_create_cq (3), -.BR ibv_destroy_cq (3), -.BR ibv_resize_cq (3), -.BR ibv_req_notify_cq (3), -.BR ibv_ack_cq_events (3), -.BR ibv_create_qp (3) -.SH "AUTHORS" -.TP -Matan Barak <matanb@mellanox.com> diff --git a/man/ibv_create_flow.3 b/man/ibv_create_flow.3 deleted file mode 100644 index df6ddd3..0000000 --- a/man/ibv_create_flow.3 +++ /dev/null @@ -1,174 +0,0 @@ -.TH IBV_CREATE_FLOW 3 2016-03-15 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_flow, ibv_destroy_flow \- create or destroy flow steering rules -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_flow *ibv_create_flow(struct ibv_qp " "*qp" , -.BI " struct ibv_flow_attr " "*flow_attr"); -.BI "int ibv_destroy_flow(struct ibv_flow " "*flow_id"); -.sp -.fi -.SH "DESCRIPTION" -.SS ibv_create_flow() -allows a user application QP -.I qp -to be attached into a specified flow -.I flow -which is defined in -.I <infiniband/verbs.h> -.PP -.nf -struct ibv_flow_attr { -.in +8 -uint32_t comp_mask; /* Future extendibility */ -enum ibv_flow_attr_type type; /* Rule type - see below */ -uint16_t size; /* Size of command */ -uint16_t priority; /* Rule priority - see below */ -uint8_t num_of_specs; /* Number of ibv_flow_spec_xxx */ -uint8_t port; /* The uplink port number */ -uint32_t flags; /* Extra flags for rule - see below */ -/* Following are the optional layers according to user request - * struct ibv_flow_spec_xxx - * struct ibv_flow_spec_yyy - */ -.in -8 -}; -.sp -.nf -enum ibv_flow_attr_type { -.in +8 -IBV_FLOW_ATTR_NORMAL = 0x0, /* Steering according to rule specifications */ -IBV_FLOW_ATTR_ALL_DEFAULT = 0x1, /* Default unicast and multicast rule - receive all Eth traffic which isn't steered to any QP */ -IBV_FLOW_ATTR_MC_DEFAULT = 0x2, /* Default multicast rule - receive all Eth multicast traffic which isn't steered to any QP */ -.in -8 -}; -.sp -.nf -enum ibv_flow_flags { -.in +8 -IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1 << 0, /* Apply the rules on packets that were sent from the attached QP through loopback */ -IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1, /* Rule doesn't trap received packets, allowing them to match lower prioritized rules */ -.in -8 -}; -.fi -.PP -Each spec struct holds the relevant network layer parameters for matching. To enforce the match, the user sets a mask for each parameter. -.br -If the bit is set in the mask, the corresponding bit in the value should be matched. -.br -Note that most vendors support either full mask (all "1"s) or zero mask (all "0"s). -.br -.B Network parameters in the relevant network structs should be given in network order (big endian). - -.SS Flow domains and priority -Flow steering defines the concept of domain and priority. Each domain represents an application that can attach a flow. -Domains are prioritized. A higher priority domain will always supersede a lower priority domain when their flow specifications overlap. -.br -.B IB verbs have the higher priority domain. -.br -In addition to the domain, there is priority within each of the domains. -A lower priority numeric value (higher priority) takes precedence over matching rules with higher numeric priority value (lower priority). -It is important to note that the priority value of a flow spec is used not only to establish the precedence of conflicting flow matches -but also as a way to abstract the order on which flow specs are tested for matches. Flows with higher priorities will be tested before flows with lower priorities. -.PP -.SS ibv_destroy_flow() -destroys the flow -.I flow_id\fR. -.SH "RETURN VALUE" -.B ibv_create_flow() -returns a pointer to the flow, or NULL if the request fails. In case of an error, errno is updated. -.PP -.B ibv_destroy_flow() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "ERRORS" -.SS EINVAL -.B ibv_create_flow() -flow specification, QP or priority are invalid -.PP -.B ibv_destroy_flow() -flow_id is invalid -.SS ENOMEM -Couldn't create/destroy flow, not enough memory -.SS ENXIO -Device managed flow steering isn't currently supported -.SS EPERM -No permissions to add the flow steering rule -.SH "NOTES" -These verbs are available only for devices supporting -.br -IBV_DEVICE_MANAGED_FLOW_STEERING and only for QPs of Transport Service Type -.BR IBV_QPT_UD -or -.BR IBV_QPT_RAW_PACKET -.PP -.SH EXAMPLE -.br -Below flow_attr defines a rule in priority 0 to match a destination -mac address and a source ipv4 address. For that, L2 and L3 specs are used. -.br -If there is a hit on this rule, means the -received packet has destination mac: 66:11:22:33:44:55 and source ip: 0x0B86C806, -the packet is steered to its attached qp. -.sp -.nf -struct raw_eth_flow_attr { -.in +8 -struct ibv_flow_attr attr; -struct ibv_flow_spec_eth spec_eth; -struct ibv_flow_spec_ipv4 spec_ipv4; -.in -8 -} __attribute__((packed)); -.sp -.nf -struct raw_eth_flow_attr flow_attr = { -.in +8 - .attr = { - .comp_mask = 0, - .type = IBV_FLOW_ATTR_NORMAL, - .size = sizeof(flow_attr), - .priority = 0, - .num_of_specs = 2, - .port = 1, - .flags = 0, - }, - .spec_eth = { - .type = IBV_FLOW_SPEC_ETH, - .size = sizeof(struct ibv_flow_spec_eth), - .val = { - .dst_mac = {0x66, 0x11, 0x22, 0x33, 0x44, 0x55}, - .src_mac = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - .ether_type = 0, - .vlan_tag = 0, - }, - .mask = { - .dst_mac = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - .src_mac = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - .ether_type = 0, - .vlan_tag = 0, - } - }, - .spec_ipv4 = { - .type = IBV_FLOW_SPEC_IPV4, - .size = sizeof(struct ibv_flow_spec_ipv4), - .val = { - .src_ip = 0x0B86C806, - .dst_ip = 0, - }, - .mask = { - .src_ip = 0xFFFFFFFF, - .dst_ip = 0, - } - } -.in -8 -}; -.sp -.nf -.SH "AUTHORS" -.TP -Hadar Hen Zion <hadarh@mellanox.com> -.TP -Matan Barak <matanb@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_create_qp.3 b/man/ibv_create_qp.3 deleted file mode 100644 index 7feeab2..0000000 --- a/man/ibv_create_qp.3 +++ /dev/null @@ -1,85 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_QP 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_qp, ibv_destroy_qp \- create or destroy a queue pair (QP) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_qp *ibv_create_qp(struct ibv_pd " "*pd" , -.BI " struct ibv_qp_init_attr " "*qp_init_attr" ); -.sp -.BI "int ibv_destroy_qp(struct ibv_qp " "*qp" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_qp() -creates a queue pair (QP) associated with the protection domain -.I pd\fR. -The argument -.I qp_init_attr -is an ibv_qp_init_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_qp_init_attr { -.in +8 -void *qp_context; /* Associated context of the QP */ -struct ibv_cq *send_cq; /* CQ to be associated with the Send Queue (SQ) */ -struct ibv_cq *recv_cq; /* CQ to be associated with the Receive Queue (RQ) */ -struct ibv_srq *srq; /* SRQ handle if QP is to be associated with an SRQ, otherwise NULL */ -struct ibv_qp_cap cap; /* QP capabilities */ -enum ibv_qp_type qp_type; /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, IBV_QPT_UD or IBV_QPT_RAW_PACKET */ -int sq_sig_all; /* If set, each Work Request (WR) submitted to the SQ generates a completion entry */ -.in -8 -}; -.sp -.nf -struct ibv_qp_cap { -.in +8 -uint32_t max_send_wr; /* Requested max number of outstanding WRs in the SQ */ -uint32_t max_recv_wr; /* Requested max number of outstanding WRs in the RQ */ -uint32_t max_send_sge; /* Requested max number of scatter/gather (s/g) elements in a WR in the SQ */ -uint32_t max_recv_sge; /* Requested max number of s/g elements in a WR in the SQ */ -uint32_t max_inline_data;/* Requested max number of data (bytes) that can be posted inline to the SQ, otherwise 0 */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_qp() -will update the -.I qp_init_attr\fB\fR->cap -struct with the actual \s-1QP\s0 values of the QP that was created; -the values will be greater than or equal to the values requested. -.PP -.B ibv_destroy_qp() -destroys the QP -.I qp\fR. -.SH "RETURN VALUE" -.B ibv_create_qp() -returns a pointer to the created QP, or NULL if the request fails. -Check the QP number (\fBqp_num\fR) in the returned QP. -.PP -.B ibv_destroy_qp() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_create_qp() -will fail if a it is asked to create QP of a type other than -.B IBV_QPT_RC -or -.B IBV_QPT_UD -associated with an SRQ. -.PP -The attributes max_recv_wr and max_recv_sge are ignored by -.B ibv_create_qp() -if the QP is to be associated with an SRQ. -.PP -.B ibv_destroy_qp() -fails if the QP is attached to a multicast group. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_modify_qp (3), -.BR ibv_query_qp (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_qp_ex.3 b/man/ibv_create_qp_ex.3 deleted file mode 100644 index 08cb92c..0000000 --- a/man/ibv_create_qp_ex.3 +++ /dev/null @@ -1,96 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_QP_EX 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_qp_ex, ibv_destroy_qp \- create or destroy a queue pair (QP) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_qp *ibv_create_qp_ex(struct ibv_context " "*context" , -.BI " struct ibv_qp_init_attr_ex " "*qp_init_attr" ); -.sp -.BI "int ibv_destroy_qp(struct ibv_qp " "*qp" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_qp_ex() -creates a queue pair (QP) associated with the protection domain -.I pd\fR. -The argument -.I qp_init_attr_ex -is an ibv_qp_init_attr_ex struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_qp_init_attr_ex { -.in +8 -void *qp_context; /* Associated context of the QP */ -struct ibv_cq *send_cq; /* CQ to be associated with the Send Queue (SQ) */ -struct ibv_cq *recv_cq; /* CQ to be associated with the Receive Queue (RQ) */ -struct ibv_srq *srq; /* SRQ handle if QP is to be associated with an SRQ, otherwise NULL */ -struct ibv_qp_cap cap; /* QP capabilities */ -enum ibv_qp_type qp_type; /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, IBV_QPT_UD or IBV_QPT_RAW_PACKET */ -int sq_sig_all; /* If set, each Work Request (WR) submitted to the SQ generates a completion entry */ -uint32_t comp_mask; /* Identifies valid fields */ -struct ibv_pd *pd; /* PD to be associated with the QP */ -struct ibv_xrcd *xrcd; /* XRC domain to be associated with the target QP */ -enum ibv_qp_create_flags create_flags; /* Creation flags for this QP */ -uint16_t max_tso_header; /* Maximum TSO header size */ -struct ibv_rwq_ind_table *rwq_ind_tbl; /* Indirection table to be associated with the QP */ -struct ibv_rx_hash_conf rx_hash_conf; /* RX hash configuration to be used */ -.in -8 -}; -.sp -.nf -struct ibv_qp_cap { -.in +8 -uint32_t max_send_wr; /* Requested max number of outstanding WRs in the SQ */ -uint32_t max_recv_wr; /* Requested max number of outstanding WRs in the RQ */ -uint32_t max_send_sge; /* Requested max number of scatter/gather (s/g) elements in a WR in the SQ */ -uint32_t max_recv_sge; /* Requested max number of s/g elements in a WR in the SQ */ -uint32_t max_inline_data;/* Requested max number of data (bytes) that can be posted inline to the SQ, otherwise 0 */ -.in -8 -}; -.nf -struct ibv_rx_hash_conf { -.in +8 -uint8_t rx_hash_function; /* RX hash function, use enum ibv_rx_hash_function_flags */ -uint8_t rx_hash_key_len; /* RX hash key length */ -uint8_t *rx_hash_key; /* RX hash key data */ -uint64_t rx_hash_fields_mask; /* RX fields that should participate in the hashing, use enum ibv_rx_hash_fields */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_qp_ex() -will update the -.I qp_init_attr_ex\fB\fR->cap -struct with the actual \s-1QP\s0 values of the QP that was created; -the values will be greater than or equal to the values requested. -.PP -.B ibv_destroy_qp() -destroys the QP -.I qp\fR. -.SH "RETURN VALUE" -.B ibv_create_qp_ex() -returns a pointer to the created QP, or NULL if the request fails. -Check the QP number (\fBqp_num\fR) in the returned QP. -.PP -.B ibv_destroy_qp() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.PP -The attributes max_recv_wr and max_recv_sge are ignored by -.B ibv_create_qp_ex() -if the QP is to be associated with an SRQ. -.PP -.B ibv_destroy_qp() -fails if the QP is attached to a multicast group. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_modify_qp (3), -.BR ibv_query_qp (3), -.BR ibv_create_rwq_ind_table (3) -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_create_rwq_ind_table.3 b/man/ibv_create_rwq_ind_table.3 deleted file mode 100644 index 2b060b5..0000000 --- a/man/ibv_create_rwq_ind_table.3 +++ /dev/null @@ -1,58 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH CREATE_RWQ_IND_TBL 3 2016-07-27 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_rwq_ind_table, ibv_destroy_rwq_ind_table \- create or destroy a Receive Work Queue Indirection Table (RWQ IND TBL). -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_rwq_ind_table *ibv_create_rwq_ind_table(struct ibv_context " "*context," -.BI " struct ibv_rwq_ind_table_init_attr " "*init_attr" ); -.sp -.BI "int ibv_destroy_rwq_ind_table(struct ibv_rwq_ind_table " "*rwq_ind_table" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_rwq_ind_table() -creates a RWQ IND TBL associated with the ibv_context -.I context\fR. -The argument -.I init_attr -is an ibv_rwq_ind_table_init_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_rwq_ind_table_init_attr { -.in +8 -uint32_t log_ind_tbl_size; /* Log, base 2, of Indirection table size */ -struct ibv_wq **ind_tbl; /* Each entry is a pointer to Receive Work Queue */ -uint32_t comp_mask; /* Identifies valid fields. Use ibv_ind_table_init_attr_mask */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_rwq_ind_table() -will create a RWQ IND TBL that holds a table of Receive Work Queue. -For further usage of the created object see below -.I NOTES\fR. -.PP -.B ibv_destroy_rwq_ind_table() -destroys the RWQ IND TBL -.I rwq_ind_table\fR. -.SH "RETURN VALUE" -.B ibv_create_rwq_ind_table() -returns a pointer to the created RWQ IND TBL, or NULL if the request fails. -.PP -.B ibv_destroy_rwq_ind_table() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The created object should be used as part of -.I ibv_create_qp_ex() -to enable dispatching of incoming packets based on some RX hash configuration. -.SH "SEE ALSO" -.BR ibv_create_wq (3), -.BR ibv_modify_wq (3), -.BR ibv_create_qp_ex (3), -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_create_srq.3 b/man/ibv_create_srq.3 deleted file mode 100644 index 7a826a1..0000000 --- a/man/ibv_create_srq.3 +++ /dev/null @@ -1,67 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_SRQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_srq, ibv_destroy_srq \- create or destroy a shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_srq *ibv_create_srq(struct ibv_pd " "*pd" ", struct " -.BI " ibv_srq_init_attr " "*srq_init_attr" ); -.sp -.BI "int ibv_destroy_srq(struct ibv_srq " "*srq" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_srq() -creates a shared receive queue (SRQ) associated with the protection domain -.I pd\fR. -The argument -.I srq_init_attr -is an ibv_srq_init_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_srq_init_attr { -.in +8 -void *srq_context; /* Associated context of the SRQ */ -struct ibv_srq_attr attr; /* SRQ attributes */ -.in -8 -}; -.sp -.nf -struct ibv_srq_attr { -.in +8 -uint32_t max_wr; /* Requested max number of outstanding work requests (WRs) in the SRQ */ -uint32_t max_sge; /* Requested max number of scatter elements per WR */ -uint32_t srq_limit; /* The limit value of the SRQ (irrelevant for ibv_create_srq) */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_srq() -will update the -.I srq_init_attr -struct with the original values of the SRQ that was created; the -values of max_wr and max_sge will be greater than or equal to the -values requested. -.PP -.B ibv_destroy_srq() -destroys the SRQ -.I srq\fR. -.SH "RETURN VALUE" -.B ibv_create_srq() -returns a pointer to the created SRQ, or NULL if the request fails. -.PP -.B ibv_destroy_srq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_destroy_srq() -fails if any queue pair is still associated with this SRQ. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_modify_srq (3), -.BR ibv_query_srq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_create_srq_ex.3 b/man/ibv_create_srq_ex.3 deleted file mode 100644 index 9151d6e..0000000 --- a/man/ibv_create_srq_ex.3 +++ /dev/null @@ -1,71 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_SRQ_EX 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_srq_ex, ibv_destroy_srq \- create or destroy a shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_srq *ibv_create_srq_ex(struct ibv_context " "*context" ", struct " -.BI " ibv_srq_init_attr_ex " "*srq_init_attr_ex" ); -.sp -.BI "int ibv_destroy_srq(struct ibv_srq " "*srq" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_srq_ex() -creates a shared receive queue (SRQ) supporting both basic and xrc modes. -The argument -.I srq_init_attr_ex -is an ibv_srq_init_attr_ex struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_srq_init_attr_ex { -.in +8 -void *srq_context; /* Associated context of the SRQ */ -struct ibv_srq_attr attr; /* SRQ attributes */ -uint32_t comp_mask; /* Identifies valid fields */ -enum ibv_srq_type srq_type; /* Basic / XRC */ -struct ibv_pd *pd; /* PD associated with the SRQ */ -struct ibv_xrcd *xrcd; /* XRC domain to associate with the SRQ */ -struct ibv_cq *cq; /* CQ to associate with the SRQ for XRC mode */ -.in -8 -}; -.sp -.nf -struct ibv_srq_attr { -.in +8 -uint32_t max_wr; /* Requested max number of outstanding work requests (WRs) in the SRQ */ -uint32_t max_sge; /* Requested max number of scatter elements per WR */ -uint32_t srq_limit; /* The limit value of the SRQ */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_srq_ex() -will update the -.I srq_init_attr_ex -struct with the original values of the SRQ that was created; the -values of max_wr and max_sge will be greater than or equal to the -values requested. -.PP -.B ibv_destroy_srq() -destroys the SRQ -.I srq\fR. -.SH "RETURN VALUE" -.B ibv_create_srq_ex() -returns a pointer to the created SRQ, or NULL if the request fails. -.PP -.B ibv_destroy_srq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_destroy_srq() -fails if any queue pair is still associated with this SRQ. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_modify_srq (3), -.BR ibv_query_srq (3) -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_create_wq.3 b/man/ibv_create_wq.3 deleted file mode 100644 index 1ee584a..0000000 --- a/man/ibv_create_wq.3 +++ /dev/null @@ -1,59 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_CREATE_WQ 3 2016-07-27 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_create_wq, ibv_destroy_wq \- create or destroy a Work Queue (WQ). -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs_exp.h> -.sp -.BI "struct ibv_wq *ibv_create_wq(struct ibv_context " "*context," -.BI " struct ibv_wq_init_attr " "*wq_init_attr" ); -.sp -.BI "int ibv_destroy_wq(struct ibv_wq " "*wq" ); -.fi -.SH "DESCRIPTION" -.B ibv_create_wq() -creates a WQ associated with the ibv_context -.I context\fR. -The argument -.I wq_init_attr -is an ibv_wq_init_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_wq_init_attr { -.in +8 -void *wq_context; /* Associated context of the WQ */ -enum ibv_wq_type wq_type; /* WQ type */ -uint32_t max_wr; /* Requested max number of outstanding WRs in the WQ */ -uint32_t max_sge; /* Requested max number of scatter/gather (s/g) elements per WR in the WQ */ -struct ibv_pd *pd; /* PD to be associated with the WQ */ -struct ibv_cq *cq; /* CQ to be associated with the WQ */ -uint32_t comp_mask; /* Identifies valid fields. Use ibv_wq_init_attr_mask */ -.in -8 -}; -.fi -.PP -The function -.B ibv_create_wq() -will update the -.I wq_init_attr\fB\fR->max_wr -and -.I wq_init_attr\fB\fR->max_sge -fields with the actual \s-1WQ\s0 values of the WQ that was created; -the values will be greater than or equal to the values requested. -.PP -.B ibv_destroy_wq() -destroys the WQ -.I wq\fR. -.SH "RETURN VALUE" -.B ibv_create_wq() -returns a pointer to the created WQ, or NULL if the request fails. -.PP -.B ibv_destroy_wq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "SEE ALSO" -.BR ibv_modify_wq (3), -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_devices.1 b/man/ibv_devices.1 deleted file mode 100644 index 99b27e5..0000000 --- a/man/ibv_devices.1 +++ /dev/null @@ -1,19 +0,0 @@ -.TH IBV_DEVICES 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_devices \- list RDMA devices - -.SH SYNOPSIS -.B ibv_devices - -.SH DESCRIPTION -.PP -List RDMA devices available for use from userspace. - -.SH SEE ALSO -.BR ibv_devinfo (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > diff --git a/man/ibv_devinfo.1 b/man/ibv_devinfo.1 deleted file mode 100644 index 70f7ed2..0000000 --- a/man/ibv_devinfo.1 +++ /dev/null @@ -1,39 +0,0 @@ -.TH IBV_DEVINFO 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_devinfo \- query RDMA devices - -.SH SYNOPSIS -.B ibv_devinfo -[\-d device] [\-i port] [\-l] [\-v] - -.SH DESCRIPTION -.PP -Print information about RDMA devices available for use from userspace. - -.SH OPTIONS - -.PP -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) - -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -query port \fIPORT\fR (default all ports) - -\fB\-l\fR, \fB\-\-list\fR -only list names of RDMA devices - -\fB\-v\fR, \fB\-\-verbose\fR -print all available information about RDMA devices - -.SH SEE ALSO -.BR ibv_devices (1) - -.SH AUTHORS -.TP -Dotan Barak -.RI < dotanba@gmail.com > -.TP -Roland Dreier -.RI < rolandd@cisco.com > diff --git a/man/ibv_event_type_str.3 b/man/ibv_event_type_str.3 deleted file mode 100644 index f9ebc35..0000000 --- a/man/ibv_event_type_str.3 +++ /dev/null @@ -1,40 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_EVENT_TYPE_STR 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -.nf -ibv_event_type_str \- Return string describing event_type enum value -.sp -ibv_node_type_str \- Return string describing node_type enum value -.sp -ibv_port_state_str \- Return string describing port_state enum value -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "const char *ibv_event_type_str(enum ibv_event_type " "event_type"); -.sp -.BI "const char *ibv_node_type_str(enum ibv_node_type " "node_type"); -.sp -.BI "const char *ibv_port_state_str(enum ibv_port_state " "port_state"); -.fi -.SH "DESCRIPTION" -.B ibv_node_type_str() -returns a string describing the node type enum value -.IR node_type . -.PP -.B ibv_port_state_str() -returns a string describing the port state enum value -.IR port_state . -.PP -.B ibv_event_type_str() -returns a string describing the event type enum value -.IR event_type . -.SH "RETURN VALUE" -These functions return a constant string that describes the enum value -passed as their argument. -.SH "AUTHOR" -.TP -Roland Dreier -.RI < rolandd@cisco.com > - diff --git a/man/ibv_fork_init.3 b/man/ibv_fork_init.3 deleted file mode 100644 index acffe3c..0000000 --- a/man/ibv_fork_init.3 +++ /dev/null @@ -1,68 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_FORK_INIT 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_fork_init \- initialize libibverbs to support fork() -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_fork_init(void); -.fi -.SH "DESCRIPTION" -.B ibv_fork_init() -initializes libibverbs's data structures to handle -.B fork() -function calls correctly and avoid data corruption, whether -.B fork() -is called explicitly or implicitly (such as in -.B system()\fR). -.PP -It is not necessary to use this function if all parent process threads -are always blocked until all child processes end or change address -spaces via an -.B exec() -operation. -.SH "RETURN VALUE" -.B ibv_fork_init() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_fork_init() -works on Linux kernels supporting the -.BR MADV_DONTFORK -flag for -.B madvise() -(2.6.17 and higher). -.PP -Setting the environment variable -.BR RDMAV_FORK_SAFE -or -.BR IBV_FORK_SAFE -has the same effect as calling -.B ibv_fork_init()\fR. -.PP -Setting the environment variable -.BR RDMAV_HUGEPAGES_SAFE -tells the library to check the underlying page size used by the kernel -for memory regions. This is required if an application uses huge -pages either directly or indirectly via a library such as libhugetlbfs. -.PP -Calling -.B ibv_fork_init() -will reduce performance due to an extra system call for every memory -registration, and the additional memory allocated to track memory -regions. The precise performance impact depends on the workload and -usually will not be significant. -.PP -Setting -.BR RDMAV_HUGEPAGES_SAFE -adds further overhead to all memory registrations. -.SH "SEE ALSO" -.BR fork (2), -.BR wait (2), -.BR system (3), -.BR exec (3), -.BR ibv_get_device_list (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_get_async_event.3 b/man/ibv_get_async_event.3 deleted file mode 100644 index a76dc0c..0000000 --- a/man/ibv_get_async_event.3 +++ /dev/null @@ -1,164 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_ASYNC_EVENT 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_async_event, ibv_ack_async_event \- get or acknowledge asynchronous events -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_get_async_event(struct ibv_context " "*context" , -.BI " struct ibv_async_event " "*event" ); -.sp -.BI "void ibv_ack_async_event(struct ibv_async_event " "*event" ); -.fi -.SH "DESCRIPTION" -.B ibv_get_async_event() -waits for the next async event of the RDMA device context -.I context -and returns it through the pointer -.I event\fR, -which is an ibv_async_event struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_async_event { -.in +8 -union { -.in +8 -struct ibv_cq *cq; /* CQ that got the event */ -struct ibv_qp *qp; /* QP that got the event */ -struct ibv_srq *srq; /* SRQ that got the event */ -int port_num; /* port number that got the event */ -.in -8 -} element; -enum ibv_event_type event_type; /* type of the event */ -.in -8 -}; -.fi -.PP -One member of the element union will be valid, depending on the -event_type member of the structure. event_type will be one of the -following events: -.PP -.I QP events: -.TP -.B IBV_EVENT_QP_FATAL \fR Error occurred on a QP and it transitioned to error state -.TP -.B IBV_EVENT_QP_REQ_ERR \fR Invalid Request Local Work Queue Error -.TP -.B IBV_EVENT_QP_ACCESS_ERR \fR Local access violation error -.TP -.B IBV_EVENT_COMM_EST \fR Communication was established on a QP -.TP -.B IBV_EVENT_SQ_DRAINED \fR Send Queue was drained of outstanding messages in progress -.TP -.B IBV_EVENT_PATH_MIG \fR A connection has migrated to the alternate path -.TP -.B IBV_EVENT_PATH_MIG_ERR \fR A connection failed to migrate to the alternate path -.TP -.B IBV_EVENT_QP_LAST_WQE_REACHED \fR Last WQE Reached on a QP associated with an SRQ -.PP -.I CQ events: -.TP -.B IBV_EVENT_CQ_ERR \fR CQ is in error (CQ overrun) -.PP -.I SRQ events: -.TP -.B IBV_EVENT_SRQ_ERR \fR Error occurred on an SRQ -.TP -.B IBV_EVENT_SRQ_LIMIT_REACHED \fR SRQ limit was reached -.PP -.I Port events: -.TP -.B IBV_EVENT_PORT_ACTIVE \fR Link became active on a port -.TP -.B IBV_EVENT_PORT_ERR \fR Link became unavailable on a port -.TP -.B IBV_EVENT_LID_CHANGE \fR LID was changed on a port -.TP -.B IBV_EVENT_PKEY_CHANGE \fR P_Key table was changed on a port -.TP -.B IBV_EVENT_SM_CHANGE \fR SM was changed on a port -.TP -.B IBV_EVENT_CLIENT_REREGISTER \fR SM sent a CLIENT_REREGISTER request to a port -.TP -.B IBV_EVENT_GID_CHANGE \fR GID table was changed on a port -.PP -.I CA events: -.TP -.B IBV_EVENT_DEVICE_FATAL \fR CA is in FATAL state -.PP -.B ibv_ack_async_event() -acknowledge the async event -.I event\fR. -.SH "RETURN VALUE" -.B ibv_get_async_event() -returns 0 on success, and \-1 on error. -.PP -.B ibv_ack_async_event() -returns no value. -.SH "NOTES" -All async events that -.B ibv_get_async_event() -returns must be acknowledged using -.B ibv_ack_async_event()\fR. -To avoid races, destroying an object (CQ, SRQ or QP) will wait for all -affiliated events for the object to be acknowledged; this avoids an -application retrieving an affiliated event after the corresponding -object has already been destroyed. -.PP -.B ibv_get_async_event() -is a blocking function. If multiple threads call this function -simultaneously, then when an async event occurs, only one thread will -receive it, and it is not possible to predict which thread will -receive it. -.SH "EXAMPLES" -The following code example demonstrates one possible way to work with async events in non-blocking mode. -It performs the following steps: -.PP -1. Set the async events queue work mode to be non-blocked -.br -2. Poll the queue until it has an async event -.br -3. Get the async event and ack it -.PP -.nf -/* change the blocking mode of the async event queue */ -flags = fcntl(ctx->async_fd, F_GETFL); -rc = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK); -if (rc < 0) { - fprintf(stderr, "Failed to change file descriptor of async event queue\en"); - return 1; -} - -/* - * poll the queue until it has an event and sleep ms_timeout - * milliseconds between any iteration - */ -my_pollfd.fd = ctx->async_fd; -my_pollfd.events = POLLIN; -my_pollfd.revents = 0; - -do { - rc = poll(&my_pollfd, 1, ms_timeout); -} while (rc == 0); -if (rc < 0) { - fprintf(stderr, "poll failed\en"); - return 1; -} - -/* Get the async event */ -if (ibv_get_async_event(ctx, &async_event)) { - fprintf(stderr, "Failed to get async_event\en"); - return 1; -} - -/* Ack the event */ -ibv_ack_async_event(&async_event); - -.fi -.SH "SEE ALSO" -.BR ibv_open_device (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_get_cq_event.3 b/man/ibv_get_cq_event.3 deleted file mode 100644 index 70b2572..0000000 --- a/man/ibv_get_cq_event.3 +++ /dev/null @@ -1,185 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_CQ_EVENT 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_cq_event, ibv_ack_cq_events \- get and acknowledge completion queue (CQ) events - -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_get_cq_event(struct ibv_comp_channel " "*channel" , -.BI " struct ibv_cq " "**cq" ", void " "**cq_context" ); -.sp -.BI "void ibv_ack_cq_events(struct ibv_cq " "*cq" ", unsigned int " "nevents" ); -.fi - -.SH "DESCRIPTION" -.B ibv_get_cq_event() -waits for the next completion event in the completion event channel -.I channel\fR. -Fills the arguments -.I cq -with the CQ that got the event and -.I cq_context -with the CQ's context\fR. -.PP -.B ibv_ack_cq_events() -acknowledges -.I nevents -events on the CQ -.I cq\fR. - -.SH "RETURN VALUE" -.B ibv_get_cq_event() -returns 0 on success, and \-1 on error. -.PP -.B ibv_ack_cq_events() -returns no value. -.SH "NOTES" -All completion events that -.B ibv_get_cq_event() -returns must be acknowledged using -.B ibv_ack_cq_events()\fR. -To avoid races, destroying a CQ will wait for all completion events to -be acknowledged; this guarantees a one-to-one correspondence between -acks and successful gets. -.PP -Calling -.B ibv_ack_cq_events() -may be relatively expensive in the datapath, since it must take a -mutex. Therefore it may be better to amortize this cost by -keeping a count of the number of events needing acknowledgement and -acking several completion events in one call to -.B ibv_ack_cq_events()\fR. -.SH "EXAMPLES" -The following code example demonstrates one possible way to work with -completion events. It performs the following steps: -.PP -Stage I: Preparation -.br -1. Creates a CQ -.br -2. Requests for notification upon a new (first) completion event -.PP -Stage II: Completion Handling Routine -.br -3. Wait for the completion event and ack it -.br -4. Request for notification upon the next completion event -.br -5. Empty the CQ -.PP -Note that an extra event may be triggered without having a -corresponding completion entry in the CQ. This occurs if a completion -entry is added to the CQ between Step 4 and Step 5, and the CQ is then -emptied (polled) in Step 5. -.PP -.nf -cq = ibv_create_cq(ctx, 1, ev_ctx, channel, 0); -if (!cq) { - fprintf(stderr, "Failed to create CQ\en"); - return 1; -} -.PP -/* Request notification before any completion can be created */ -if (ibv_req_notify_cq(cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\en"); - return 1; -} -.PP -\&. -\&. -\&. -.PP -/* Wait for the completion event */ -if (ibv_get_cq_event(channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\en"); - return 1; -} - -/* Ack the event */ -ibv_ack_cq_events(ev_cq, 1); -.PP -/* Request notification upon the next completion event */ -if (ibv_req_notify_cq(ev_cq, 0)) { - fprintf(stderr, "Couldn't request CQ notification\en"); - return 1; -} -.PP -/* Empty the CQ: poll all of the completions from the CQ (if any exist) */ -do { - ne = ibv_poll_cq(cq, 1, &wc); - if (ne < 0) { - fprintf(stderr, "Failed to poll completions from the CQ\en"); - return 1; - } - - /* there may be an extra event with no completion in the CQ */ - if (ne == 0) - continue; -.PP - if (wc.status != IBV_WC_SUCCESS) { - fprintf(stderr, "Completion with status 0x%x was found\en", wc.status); - return 1; - } -} while (ne); -.fi - -The following code example demonstrates one possible way to work with -completion events in non-blocking mode. It performs the following -steps: -.PP -1. Set the completion event channel to be non-blocked -.br -2. Poll the channel until there it has a completion event -.br -3. Get the completion event and ack it -.PP -.nf -/* change the blocking mode of the completion channel */ -flags = fcntl(channel->fd, F_GETFL); -rc = fcntl(channel->fd, F_SETFL, flags | O_NONBLOCK); -if (rc < 0) { - fprintf(stderr, "Failed to change file descriptor of completion event channel\en"); - return 1; -} - - -/* - * poll the channel until it has an event and sleep ms_timeout - * milliseconds between any iteration - */ -my_pollfd.fd = channel->fd; -my_pollfd.events = POLLIN; -my_pollfd.revents = 0; - -do { - rc = poll(&my_pollfd, 1, ms_timeout); -} while (rc == 0); -if (rc < 0) { - fprintf(stderr, "poll failed\en"); - return 1; -} -ev_cq = cq; - -/* Wait for the completion event */ -if (ibv_get_cq_event(channel, &ev_cq, &ev_ctx)) { - fprintf(stderr, "Failed to get cq_event\en"); - return 1; -} - -/* Ack the event */ -ibv_ack_cq_events(ev_cq, 1); - -.fi -.SH "SEE ALSO" -.BR ibv_create_comp_channel (3), -.BR ibv_create_cq (3), -.BR ibv_req_notify_cq (3), -.BR ibv_poll_cq (3) - -.SH "AUTHORS" -.TP -Dotan Barak -.RI < dotanba@gmail.com > diff --git a/man/ibv_get_device_guid.3 b/man/ibv_get_device_guid.3 deleted file mode 100644 index 8cbe0e7..0000000 --- a/man/ibv_get_device_guid.3 +++ /dev/null @@ -1,25 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_DEVICE_GUID 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_device_guid \- get an RDMA device's GUID -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "uint64_t ibv_get_device_guid(struct ibv_device " "*device" "); -.fi -.SH "DESCRIPTION" -.B ibv_get_device_name() -returns the Global Unique IDentifier (GUID) of the RDMA device -.I device\fR. -.SH "RETURN VALUE" -.B ibv_get_device_guid() -returns the GUID of the device in network byte order. -.SH "SEE ALSO" -.BR ibv_get_device_list (3), -.BR ibv_get_device_name (3), -.BR ibv_open_device (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_get_device_list.3 b/man/ibv_get_device_list.3 deleted file mode 100644 index 96de554..0000000 --- a/man/ibv_get_device_list.3 +++ /dev/null @@ -1,66 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_DEVICE_LIST 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_device_list, ibv_free_device_list \- get and release list of available RDMA devices -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_device **ibv_get_device_list(int " "*num_devices" ); -.sp -.BI "void ibv_free_device_list(struct ibv_device " "**list" ); -.fi -.SH "DESCRIPTION" -.B ibv_get_device_list() -returns a NULL-terminated array of RDMA devices currently available. -The argument -.I num_devices -is optional; if not NULL, it is set to the number of devices returned in the array. -.PP -.B ibv_free_device_list() -frees the array of devices -.I list -returned by -.B ibv_get_device_list()\fR. -.SH "RETURN VALUE" -.B ibv_get_device_list() -returns the array of available RDMA devices, or sets -.I errno -and returns NULL if the request fails. If no devices are found then -.I num_devices -is set to 0, and non-NULL is returned. -.PP -.B ibv_free_device_list() -returns no value. -.SH "ERRORS" -.TP -.B EPERM -Permission denied. -.TP -.B ENOSYS -No kernel support for RDMA. -.TP -.B ENOMEM -Insufficient memory to complete the operation. -.SH "NOTES" -Client code should open all the devices it intends to use with -.B ibv_open_device()\fR before calling -.B ibv_free_device_list()\fR. -Once it frees the array with -.B ibv_free_device_list()\fR, -it will be able to use only the open devices; pointers to unopened devices will no longer be valid. -.P -Setting the environment variable -.BR IBV_SHOW_WARNINGS -will cause warnings to be emitted to stderr if a kernel verbs device -is discovered, but no corresponding userspace driver can be found for -it. -.SH "SEE ALSO" -.BR ibv_fork_init (3), -.BR ibv_get_device_name (3), -.BR ibv_get_device_guid (3), -.BR ibv_open_device (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_get_device_name.3 b/man/ibv_get_device_name.3 deleted file mode 100644 index b6cb491..0000000 --- a/man/ibv_get_device_name.3 +++ /dev/null @@ -1,25 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_DEVICE_NAME 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_device_name \- get an RDMA device's name -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "const char *ibv_get_device_name(struct ibv_device " "*device" "); -.fi -.SH "DESCRIPTION" -.B ibv_get_device_name() -returns a human-readable name associated with the RDMA device -.I device\fR. -.SH "RETURN VALUE" -.B ibv_get_device_name() -returns a pointer to the device name, or NULL if the request fails. -.SH "SEE ALSO" -.BR ibv_get_device_list (3), -.BR ibv_get_device_guid (3), -.BR ibv_open_device (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_get_srq_num.3 b/man/ibv_get_srq_num.3 deleted file mode 100644 index 00e4fb4..0000000 --- a/man/ibv_get_srq_num.3 +++ /dev/null @@ -1,32 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_GET_SRQ_NUM 3 2013-06-26 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_get_srq_num \- return srq number associated with the given shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_get_srq_num(struct ibv_srq " "*srq" , -.BI " uint32_t " "*srq_num" ); -.fi -.SH "DESCRIPTION" -.B ibv_get_srq_num() -return srq number associated with the given shared receive queue -The argument -.I srq -is an ibv_srq struct, as defined in <infiniband/verbs.h>. -.I srq_num -is an output parameter that holds the returned srq number. -.PP -.nf -.SH "RETURN VALUE" -.B ibv_get_srq_num() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_modify_srq (3), -.BR ibv_create_srq_ex (3) -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_inc_rkey.3 b/man/ibv_inc_rkey.3 deleted file mode 100644 index 930368b..0000000 --- a/man/ibv_inc_rkey.3 +++ /dev/null @@ -1,32 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_INC_RKEY 3 2015-01-29 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -.nf -ibv_inc_rkey \- creates a new rkey from the given one -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "uint32_t ibv_inc_rkey(uint32_t " "rkey" "); -.fi -.SH "DESCRIPTION" -.B ibv_inc_rkey() -Increases the 8 LSB of -.I rkey -and returns the new value. -.PP -.SH "RETURN VALUE" -.B ibv_inc_rkey() -returns the new rkey. -.SH "NOTES" -.PP -The verb generates a new rkey that is different from the previous one on its tag part -but has the same index (bits 0xffffff00). -A use case for this verb can be to create a new rkey from a Memory window's rkey -when binding it to a Memory region. -.SH "AUTHORS" -.TP -Majd Dibbiny <majd@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_modify_qp.3 b/man/ibv_modify_qp.3 deleted file mode 100644 index cb3faaa..0000000 --- a/man/ibv_modify_qp.3 +++ /dev/null @@ -1,179 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_MODIFY_QP 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_modify_qp \- modify the attributes of a queue pair (QP) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_modify_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" , -.BI " int " "attr_mask" ); -.fi -.SH "DESCRIPTION" -.B ibv_modify_qp() -modifies the attributes of QP -.I qp -with the attributes in -.I attr -according to the mask -.I attr_mask\fR. -The argument \fIattr\fR is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_qp_attr { -.in +8 -enum ibv_qp_state qp_state; /* Move the QP to this state */ -enum ibv_qp_state cur_qp_state; /* Assume this is the current QP state */ -enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */ -enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */ -uint32_t qkey; /* Q_Key for the QP (valid only for UD QPs) */ -uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */ -uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */ -uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */ -int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */ -struct ibv_qp_cap cap; /* QP capabilities (valid if HCA supports QP resizing) */ -struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */ -struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */ -uint16_t pkey_index; /* Primary P_Key index */ -uint16_t alt_pkey_index; /* Alternate P_Key index */ -uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification (Valid only if qp_state is SQD) */ -uint8_t sq_draining; /* Is the QP draining? Irrelevant for ibv_modify_qp() */ -uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */ -uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */ -uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */ -uint8_t port_num; /* Primary port number */ -uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */ -uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */ -uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */ -uint8_t alt_port_num; /* Alternate port number */ -uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */ -.in -8 -}; -.fi -.PP -For details on struct ibv_qp_cap see the description of -.B ibv_create_qp()\fR. -For details on struct ibv_ah_attr see the description of -.B ibv_create_ah()\fR. -.PP -The argument -.I attr_mask -specifies the QP attributes to be modified. -The argument is either 0 or the bitwise OR of one or more of the following flags: -.PP -.TP -.B IBV_QP_STATE \fR Modify qp_state -.TP -.B IBV_QP_CUR_STATE \fR Set cur_qp_state -.TP -.B IBV_QP_EN_SQD_ASYNC_NOTIFY \fR Set en_sqd_async_notify -.TP -.B IBV_QP_ACCESS_FLAGS \fR Set qp_access_flags -.TP -.B IBV_QP_PKEY_INDEX \fR Set pkey_index -.TP -.B IBV_QP_PORT \fR Set port_num -.TP -.B IBV_QP_QKEY \fR Set qkey -.TP -.B IBV_QP_AV \fR Set ah_attr -.TP -.B IBV_QP_PATH_MTU \fR Set path_mtu -.TP -.B IBV_QP_TIMEOUT \fR Set timeout -.TP -.B IBV_QP_RETRY_CNT \fR Set retry_cnt -.TP -.B IBV_QP_RNR_RETRY \fR Set rnr_retry -.TP -.B IBV_QP_RQ_PSN \fR Set rq_psn -.TP -.B IBV_QP_MAX_QP_RD_ATOMIC \fR Set max_rd_atomic -.TP -.B IBV_QP_ALT_PATH \fR Set the alternative path via: alt_ah_attr, alt_pkey_index, alt_port_num, alt_timeout -.TP -.B IBV_QP_MIN_RNR_TIMER \fR Set min_rnr_timer -.TP -.B IBV_QP_SQ_PSN \fR Set sq_psn -.TP -.B IBV_QP_MAX_DEST_RD_ATOMIC \fR Set max_dest_rd_atomic -.TP -.B IBV_QP_PATH_MIG_STATE \fR Set path_mig_state -.TP -.B IBV_QP_CAP \fR Set cap -.TP -.B IBV_QP_DEST_QPN \fR Set dest_qp_num -.SH "RETURN VALUE" -.B ibv_modify_qp() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -If any of the modify attributes or the modify mask are invalid, none -of the attributes will be modified (including the QP state). -.PP -Not all devices support resizing QPs. To check if a device supports it, check if the -.B IBV_DEVICE_RESIZE_MAX_WR -bit is set in the device capabilities flags. -.PP -Not all devices support alternate paths. To check if a device supports it, check if the -.B IBV_DEVICE_AUTO_PATH_MIG -bit is set in the device capabilities flags. -.PP -The following tables indicate for each QP Transport Service Type, the -minimum list of attributes that must be changed upon transitioning QP -state from: Reset \-\-> Init \-\-> RTR \-\-> RTS. -.PP -.nf -For QP Transport Service Type \fB IBV_QPT_UD\fR: -.sp -Next state Required attributes -\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -Init \fB IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR - \fB IBV_QP_QKEY \fR -RTR \fB IBV_QP_STATE \fR -RTS \fB IBV_QP_STATE, IBV_QP_SQ_PSN \fR -.fi -.PP -.nf -For QP Transport Service Type \fB IBV_QPT_UC\fR: -.sp -Next state Required attributes -\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -Init \fB IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR - \fB IBV_QP_ACCESS_FLAGS \fR -RTR \fB IBV_QP_STATE, IBV_QP_AV, IBV_QP_PATH_MTU, \fR - \fB IBV_QP_DEST_QPN, IBV_QP_RQ_PSN \fR -RTS \fB IBV_QP_STATE, IBV_QP_SQ_PSN \fR -.fi -.PP -.nf -For QP Transport Service Type \fB IBV_QPT_RC\fR: -.sp -Next state Required attributes -\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -Init \fB IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR - \fB IBV_QP_ACCESS_FLAGS \fR -RTR \fB IBV_QP_STATE, IBV_QP_AV, IBV_QP_PATH_MTU, \fR - \fB IBV_QP_DEST_QPN, IBV_QP_RQ_PSN, \fR - \fB IBV_QP_MAX_DEST_RD_ATOMIC, IBV_QP_MIN_RNR_TIMER \fR -RTS \fB IBV_QP_STATE, IBV_QP_SQ_PSN, IBV_QP_MAX_QP_RD_ATOMIC, \fR - \fB IBV_QP_RETRY_CNT, IBV_QP_RNR_RETRY, IBV_QP_TIMEOUT \fR -.fi -.PP -.nf -For QP Transport Service Type \fB IBV_QPT_RAW_PACKET\fR: -.sp -Next state Required attributes -\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -Init \fB IBV_QP_STATE, IBV_QP_PORT\fR -RTR \fB IBV_QP_STATE\fR -RTS \fB IBV_QP_STATE\fR -.fi -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_destroy_qp (3), -.BR ibv_query_qp (3), -.BR ibv_create_ah (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_modify_srq.3 b/man/ibv_modify_srq.3 deleted file mode 100644 index 9cc0497..0000000 --- a/man/ibv_modify_srq.3 +++ /dev/null @@ -1,63 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_MODIFY_SRQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_modify_srq \- modify attributes of a shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_modify_srq(struct ibv_srq " "*srq" , -.BI " struct ibv_srq_attr " "*srq_attr" , -.BI " int " "srq_attr_mask" ); -.fi -.SH "DESCRIPTION" -.B ibv_modify_srq() -modifies the attributes of SRQ -.I srq -with the attributes in -.I srq_attr -according to the mask -.I srq_attr_mask\fR. -The argument \fIsrq_attr\fR is an ibv_srq_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_srq_attr { -.in +8 -uint32_t max_wr; /* maximum number of outstanding work requests (WRs) in the SRQ */ -uint32_t max_sge; /* number of scatter elements per WR (irrelevant for ibv_modify_srq) */ -uint32_t srq_limit; /* the limit value of the SRQ */ -.in -8 -}; -.fi -.PP -The argument -.I srq_attr_mask -specifies the SRQ attributes to be modified. -The argument is either 0 or the bitwise OR of one or more of the following flags: -.PP -.TP -.B IBV_SRQ_MAX_WR \fR Resize the SRQ -.TP -.B IBV_SRQ_LIMIT \fR Set the SRQ limit -.SH "RETURN VALUE" -.B ibv_modify_srq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -If any of the modify attributes is invalid, none of the attributes will be modified. -.PP -Not all devices support resizing SRQs. To check if a device supports it, check if the -.B IBV_DEVICE_SRQ_RESIZE -bit is set in the device capabilities flags. -.PP -Modifying the srq_limit arms the SRQ to produce an -.B IBV_EVENT_SRQ_LIMIT_REACHED -"low watermark" asynchronous event once the number of WRs in the SRQ drops below srq_limit. -.SH "SEE ALSO" -.BR ibv_query_device (3), -.BR ibv_create_srq (3), -.BR ibv_destroy_srq (3), -.BR ibv_query_srq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_modify_wq.3 b/man/ibv_modify_wq.3 deleted file mode 100644 index cf37864..0000000 --- a/man/ibv_modify_wq.3 +++ /dev/null @@ -1,43 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_MODIFY_WQ 3 2016-07-27 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_modify_wq \- Modify a Work Queue (WQ). -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct int ibv_modify_wq(struct ibv_wq " "*wq," -.BI " struct ibv_wq_attr " "*wq_attr" ); -.sp -.fi -.SH "DESCRIPTION" -.B ibv_modify_wq() -modifys a WQ -.I wq\fR. -The argument -.I wq_attr -is an ibv_wq_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_wq_attr { -.in +8 -uint32_t attr_mask; /* Use enum ibv_wq_attr_mask */ -enum ibv_wq_state wq_state; /* Move to this state */ -enum ibv_wq_state curr_wq_state; /* Assume this is the current state */ -.in -8 -}; -.fi -.PP -The function -.B ibv_modify_wq() -will modify the WQ based on the given -.I wq_attr\fB\fR->attr_mask -.SH "RETURN VALUE" -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "SEE ALSO" -.BR ibv_create_wq (3), -.BR ibv_destroy_wq (3), -.SH "AUTHORS" -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_open_device.3 b/man/ibv_open_device.3 deleted file mode 100644 index ed2226c..0000000 --- a/man/ibv_open_device.3 +++ /dev/null @@ -1,43 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_OPEN_DEVICE 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_open_device, ibv_close_device \- open and close an RDMA device context -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_context *ibv_open_device(struct ibv_device " "*device" "); -.sp -.BI "int ibv_close_device(struct ibv_context " "*context" "); -.fi -.SH "DESCRIPTION" -.B ibv_open_device() -opens the device -.I device -and creates a context for further use. -.PP -.B ibv_close_device() -closes the device context -.I context\fR. -.SH "RETURN VALUE" -.B ibv_open_device() -returns a pointer to the allocated device context, or NULL if the request fails. -.PP -.B ibv_close_device() -returns 0 on success, \-1 on failure. -.SH "NOTES" -.B ibv_close_device() -does not release all the resources allocated using context -.I context\fR. -To avoid resource leaks, the user should release all associated -resources before closing a context. -.SH "SEE ALSO" -.BR ibv_get_device_list (3), -.BR ibv_query_device (3), -.BR ibv_query_port (3), -.BR ibv_query_gid (3), -.BR ibv_query_pkey (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_open_qp.3 b/man/ibv_open_qp.3 deleted file mode 100644 index 71b5d46..0000000 --- a/man/ibv_open_qp.3 +++ /dev/null @@ -1,51 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_OPEN_QP 3 2011-08-12 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_open_qp \- open a shareable queue pair (QP) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_qp *ibv_open_qp(struct ibv_context " "*context" , -.BI " struct ibv_qp_open_attr " "*qp_open_attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_open_qp() -opens an existing queue pair (QP) associated with the extended protection domain -.I xrcd\fR. -The argument -.I qp_open_attr -is an ibv_qp_open_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_qp_open_attr { -.in +8 -uint32_t comp_mask; /* Identifies valid fields */ -uint32_t qp_num; /* QP number */ -struct *ibv_xrcd; /* XRC domain */ -void *qp_context; /* User defined opaque value */ -enum ibv_qp_type qp_type; /* QP transport service type */ -.fi -.PP -.B ibv_destroy_qp() -closes the opened QP and destroys the underlying QP if it has no -other references. -.I qp\fR. -.SH "RETURN VALUE" -.B ibv_open_qp() -returns a pointer to the opened QP, or NULL if the request fails. -Check the QP number (\fBqp_num\fR) in the returned QP. -.SH "NOTES" -.B ibv_open_qp() -will fail if a it is asked to open a QP that does not exist within -the xrcd with the specified qp_num and qp_type. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_create_qp (3), -.BR ibv_create_qp_ex (3), -.BR ibv_modify_qp (3), -.BR ibv_query_qp (3) -.SH "AUTHORS" -.TP -Sean Hefty <sean.hefty@intel.com> diff --git a/man/ibv_open_xrcd.3 b/man/ibv_open_xrcd.3 deleted file mode 100644 index b9724bf..0000000 --- a/man/ibv_open_xrcd.3 +++ /dev/null @@ -1,76 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_OPEN_XRCD 3 2011-06-17 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_open_xrcd, ibv_close_xrcd \- open or close an XRC protection domain (XRCDs) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_xrcd *ibv_open_xrcd(struct ibv_context " "*context" "," -.BI " struct ibv_xrcd_init_attr " "*xrcd_init_attr" ); -.sp -.BI "int ibv_close_xrcd(struct ibv_xrcd " "*xrcd" ); -.fi -.SH "DESCRIPTION" -.B ibv_open_xrcd() -open an XRC domain for the RDMA device context -.I context -.I xrcd_init_attr -is an ibv_xrcd_init_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_xrcd_init_attr { -.in +8 -uint32_t comp_mask; /* Identifies valid fields */ -int fd; -int oflag; -.fi -.PP -.I fd -is the file descriptor to associate with the XRCD. -.I oflag -describes the desired creation attributes. It is a bitwise OR of zero or more -of the following flags: -.PP -.TP -.B O_CREAT -Indicates that an XRCD should be created and associated with the inode referenced -by the given fd. If the XRCD exists, this flag has no effect except as noted under -.BR O_EXCL -below.\fR -.TP -.B O_EXCL -If -.BR O_EXCL -and -.BR O_CREAT -are set, open will fail if an XRCD associated with the inode exists. -.PP -If -.I fd -equals -1, no inode is associated with the XRCD. To indicate that XRCD should be created, use -.I oflag -= -.B O_CREAT\fR. -.PP -.B ibv_close_xrcd() -closes the XRCD -.I xrcd\fR. -If this is the last reference, the XRCD will be destroyed. -.SH "RETURN VALUE" -.B ibv_open_xrcd() -returns a pointer to the opened XRCD, or NULL if the request fails. -.PP -.B ibv_close_xrcd() -returns 0 on success, or the value of errno on failure (which indicates the -failure reason). -.SH "NOTES" -.B ibv_close_xrcd() -may fail if any other resource is still associated with the XRCD being closed. -.SH "SEE ALSO" -.BR ibv_create_srq_ex (3), -.BR ibv_create_qp_ex (3), -.SH "AUTHORS" -.TP -Sean Hefty <sean.hefty@intel.com> diff --git a/man/ibv_poll_cq.3 b/man/ibv_poll_cq.3 deleted file mode 100644 index bbf48ff..0000000 --- a/man/ibv_poll_cq.3 +++ /dev/null @@ -1,82 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_POLL_CQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_poll_cq \- poll a completion queue (CQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_poll_cq(struct ibv_cq " "*cq" ", int " "num_entries" , -.BI " struct ibv_wc " "*wc" ); -.fi -.SH "DESCRIPTION" -.B ibv_poll_cq() -polls the CQ -.I cq -for work completions and returns the first -.I num_entries -(or all available completions if the CQ contains fewer than this number) in the array -.I wc\fR. -The argument -.I wc -is a pointer to an array of ibv_wc structs, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_wc { -.in +8 -uint64_t wr_id; /* ID of the completed Work Request (WR) */ -enum ibv_wc_status status; /* Status of the operation */ -enum ibv_wc_opcode opcode; /* Operation type specified in the completed WR */ -uint32_t vendor_err; /* Vendor error syndrome */ -uint32_t byte_len; /* Number of bytes transferred */ -uint32_t imm_data; /* Immediate data (in network byte order) */ -uint32_t qp_num; /* Local QP number of completed WR */ -uint32_t src_qp; /* Source QP number (remote QP number) of completed WR (valid only for UD QPs) */ -int wc_flags; /* Flags of the completed WR */ -uint16_t pkey_index; /* P_Key index (valid only for GSI QPs) */ -uint16_t slid; /* Source LID */ -uint8_t sl; /* Service Level */ -uint8_t dlid_path_bits; /* DLID path bits (not applicable for multicast messages) */ -.in -8 -}; -.sp -.fi -.PP -The attribute wc_flags describes the properties of the work completion. -It is either 0 or the bitwise OR of one or more of the following flags: -.PP -.TP -.B IBV_WC_GRH \fR GRH is present (valid only for UD QPs) -.TP -.B IBV_WC_WITH_IMM \fR Immediate data value is valid -.TP -.B IBV_WC_IP_CSUM_OK \fR TCP/UDP checksum over IPv4 and IPv4 header checksum are -verified. -Valid only when \fBdevice_cap_flags\fR in device_attr indicates current QP is -supported by checksum offload. -.PP -Not all -.I wc -attributes are always valid. If the completion status is other than -.B IBV_WC_SUCCESS\fR, -only the following attributes are valid: wr_id, status, qp_num, and vendor_err. -.SH "RETURN VALUE" -On success, -.B ibv_poll_cq() -returns a non-negative value equal to the number of completions -found. On failure, a negative value is returned. -.SH "NOTES" -.PP -Each polled completion is removed from the CQ and cannot be returned to it. -.PP -The user should consume work completions at a rate that prevents CQ -overrun from occurrence. In case of a CQ overrun, the async event -.B IBV_EVENT_CQ_ERR -will be triggered, and the CQ cannot be used. -.SH "SEE ALSO" -.BR ibv_post_send (3), -.BR ibv_post_recv (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_post_recv.3 b/man/ibv_post_recv.3 deleted file mode 100644 index 4e65c81..0000000 --- a/man/ibv_post_recv.3 +++ /dev/null @@ -1,76 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_POST_RECV 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_post_recv \- post a list of work requests (WRs) to a receive queue -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_post_recv(struct ibv_qp " "*qp" ", struct ibv_recv_wr " "*wr" , -.BI " struct ibv_recv_wr " "**bad_wr" ); -.fi -.SH "DESCRIPTION" -.B ibv_post_recv() -posts the linked list of work requests (WRs) starting with -.I wr -to the receive queue of the queue pair -.I qp\fR. -It stops processing WRs from this list at the first failure (that can -be detected immediately while requests are being posted), and returns -this failing WR through -.I bad_wr\fR. -.PP -The argument -.I wr -is an ibv_recv_wr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_recv_wr { -.in +8 -uint64_t wr_id; /* User defined WR ID */ -struct ibv_recv_wr *next; /* Pointer to next WR in list, NULL if last WR */ -struct ibv_sge *sg_list; /* Pointer to the s/g array */ -int num_sge; /* Size of the s/g array */ -.in -8 -}; -.sp -.nf -struct ibv_sge { -.in +8 -uint64_t addr; /* Start address of the local memory buffer */ -uint32_t length; /* Length of the buffer */ -uint32_t lkey; /* Key of the local Memory Region */ -.in -8 -}; -.fi -.SH "RETURN VALUE" -.B ibv_post_recv() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The buffers used by a WR can only be safely reused after WR the -request is fully executed and a work completion has been retrieved -from the corresponding completion queue (CQ). -.PP -If the QP -.I qp -is associated with a shared receive queue, you must use the function -.B ibv_post_srq_recv()\fR, -and not -.B ibv_post_recv()\fR, -since the QP's own receive queue will not be used. -.PP -If a WR is being posted to a UD QP, the Global Routing Header (GRH) of -the incoming message will be placed in the first 40 bytes of the -buffer(s) in the scatter list. If no GRH is present in the incoming -message, then the first bytes will be undefined. This means that in -all cases, the actual data of the incoming message will start at an -offset of 40 bytes into the buffer(s) in the scatter list. -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_post_send (3), -.BR ibv_post_srq_recv (3), -.BR ibv_poll_cq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_post_send.3 b/man/ibv_post_send.3 deleted file mode 100644 index c45eb01..0000000 --- a/man/ibv_post_send.3 +++ /dev/null @@ -1,174 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_POST_SEND 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_post_send \- post a list of work requests (WRs) to a send queue -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_post_send(struct ibv_qp " "*qp" ", struct ibv_send_wr " "*wr" , -.BI " struct ibv_send_wr " "**bad_wr" ); -.fi -.SH "DESCRIPTION" -.B ibv_post_send() -posts the linked list of work requests (WRs) starting with -.I wr -to the send queue of the queue pair -.I qp\fR. -It stops processing WRs from this list at the first failure (that can -be detected immediately while requests are being posted), and returns -this failing WR through -.I bad_wr\fR. -.PP -The argument -.I wr -is an ibv_send_wr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_send_wr { -.in +8 -uint64_t wr_id; /* User defined WR ID */ -struct ibv_send_wr *next; /* Pointer to next WR in list, NULL if last WR */ -struct ibv_sge *sg_list; /* Pointer to the s/g array */ -int num_sge; /* Size of the s/g array */ -enum ibv_wr_opcode opcode; /* Operation type */ -int send_flags; /* Flags of the WR properties */ -uint32_t imm_data; /* Immediate data (in network byte order) */ -union { -.in +8 -struct { -.in +8 -uint64_t remote_addr; /* Start address of remote memory buffer */ -uint32_t rkey; /* Key of the remote Memory Region */ -.in -8 -} rdma; -struct { -.in +8 -uint64_t remote_addr; /* Start address of remote memory buffer */ -uint64_t compare_add; /* Compare operand */ -uint64_t swap; /* Swap operand */ -uint32_t rkey; /* Key of the remote Memory Region */ -.in -8 -} atomic; -struct { -.in +8 -struct ibv_ah *ah; /* Address handle (AH) for the remote node address */ -uint32_t remote_qpn; /* QP number of the destination QP */ -uint32_t remote_qkey; /* Q_Key number of the destination QP */ -.in -8 -} ud; -.in -8 -} wr; -union { -.in +8 -struct { -.in +8 -uint32_t remote_srqn; /* Number of the remote SRQ */ -.in -8 -} xrc; -.in -8 -} qp_type; -union { -.in +8 -struct { -.in +8 -struct ibv_mw *mw; /* Memory window (MW) of type 2 to bind */ -uint32_t rkey; /* The desired new rkey of the MW */ -struct ibv_mw_bind_info bind_info; /* MW additional bind information */ -.in -8 -} bind_mw; -struct { -.in +8 -void *hdr; /* Pointer address of inline header */ -uint16_t hdr_sz; /* Inline header size */ -uint16_t mss; /* Maximum segment size for each TSO fragment */ -.in -8 -} tso; -.in -8 -}; -.in -8 -}; -.fi -.sp -.nf -struct ibv_mw_bind_info { -.in +8 -struct ibv_mr *mr; /* The Memory region (MR) to bind the MW to */ -uint64_t addr; /* The address the MW should start at */ -uint64_t length; /* The length (in bytes) the MW should span */ -int mw_access_flags; /* Access flags to the MW. Use ibv_access_flags */ -.in -8 -}; -.fi -.sp -.nf -struct ibv_sge { -.in +8 -uint64_t addr; /* Start address of the local memory buffer */ -uint32_t length; /* Length of the buffer */ -uint32_t lkey; /* Key of the local Memory Region */ -.in -8 -}; -.fi -.PP -Each QP Transport Service Type supports a specific set of opcodes, as shown in the following table: -.PP -.nf -OPCODE | IBV_QPT_UD | IBV_QPT_UC | IBV_QPT_RC | IBV_QPT_XRC_SEND | IBV_QPT_RAW_PACKET -\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -IBV_WR_SEND | X | X | X | X | X -IBV_WR_SEND_WITH_IMM | X | X | X | X | -IBV_WR_RDMA_WRITE | | X | X | X | -IBV_WR_RDMA_WRITE_WITH_IMM | | X | X | X | -IBV_WR_RDMA_READ | | | X | X | -IBV_WR_ATOMIC_CMP_AND_SWP | | | X | X | -IBV_WR_ATOMIC_FETCH_AND_ADD | | | X | X | -IBV_WR_LOCAL_INV | | X | X | X | -IBV_WR_BIND_MW | | X | X | X | -IBV_WR_SEND_WITH_INV | | X | X | X | -IBV_WR_TSO | X | | | | X -.fi -.PP -The attribute send_flags describes the properties of the \s-1WR\s0. It is either 0 or the bitwise \s-1OR\s0 of one or more of the following flags: -.PP -.TP -.B IBV_SEND_FENCE \fR Set the fence indicator. Valid only for QPs with Transport Service Type \fBIBV_QPT_RC -.TP -.B IBV_SEND_SIGNALED \fR Set the completion notification indicator. Relevant only if QP was created with sq_sig_all=0 -.TP -.B IBV_SEND_SOLICITED \fR Set the solicited event indicator. Valid only for Send and RDMA Write with immediate -.TP -.B IBV_SEND_INLINE \fR Send data in given gather list as inline data -in a send WQE. Valid only for Send and RDMA Write. The L_Key will not be checked. -.TP -.B IBV_SEND_IP_CSUM \fR Offload the IPv4 and TCP/UDP checksum calculation. -Valid only when \fBdevice_cap_flags\fR in device_attr indicates current QP is -supported by checksum offload. -.SH "RETURN VALUE" -.B ibv_post_send() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The user should not alter or destroy AHs associated with WRs until -request is fully executed and a work completion has been retrieved -from the corresponding completion queue (CQ) to avoid unexpected -behavior. -.PP -The buffers used by a WR can only be safely reused after WR the -request is fully executed and a work completion has been retrieved -from the corresponding completion queue (CQ). However, if the -IBV_SEND_INLINE flag was set, the buffer can be reused immediately -after the call returns. -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_create_ah (3), -.BR ibv_post_recv (3), -.BR ibv_post_srq_recv (3), -.BR ibv_poll_cq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> -.TP -Majd Dibbiny <majd@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_post_srq_recv.3 b/man/ibv_post_srq_recv.3 deleted file mode 100644 index 073972b..0000000 --- a/man/ibv_post_srq_recv.3 +++ /dev/null @@ -1,68 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_POST_SRQ_RECV 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_post_srq_recv \- post a list of work requests (WRs) to a shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_post_srq_recv(struct ibv_srq " "*srq" ", struct ibv_recv_wr " "*wr" , -.BI " struct ibv_recv_wr " "**bad_wr" ); -.fi -.SH "DESCRIPTION" -.B ibv_post_srq_recv() -posts the linked list of work requests (WRs) starting with -.I wr -to the shared receive queue (SRQ) -.I srq\fR. -It stops processing WRs from this list at the first failure (that can -be detected immediately while requests are being posted), and returns -this failing WR through -.I bad_wr\fR. -.PP -The argument -.I wr -is an ibv_recv_wr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_recv_wr { -.in +8 -uint64_t wr_id; /* User defined WR ID */ -struct ibv_recv_wr *next; /* Pointer to next WR in list, NULL if last WR */ -struct ibv_sge *sg_list; /* Pointer to the s/g array */ -int num_sge; /* Size of the s/g array */ -.in -8 -}; -.sp -.nf -struct ibv_sge { -.in +8 -uint64_t addr; /* Start address of the local memory buffer */ -uint32_t length; /* Length of the buffer */ -uint32_t lkey; /* Key of the local Memory Region */ -.in -8 -}; -.fi -.SH "RETURN VALUE" -.B ibv_post_srq_recv() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The buffers used by a WR can only be safely reused after WR the -request is fully executed and a work completion has been retrieved -from the corresponding completion queue (CQ). -.PP -If a WR is being posted to a UD QP, the Global Routing Header (GRH) of -the incoming message will be placed in the first 40 bytes of the -buffer(s) in the scatter list. If no GRH is present in the incoming -message, then the first bytes will be undefined. This means that in -all cases, the actual data of the incoming message will start at an -offset of 40 bytes into the buffer(s) in the scatter list. -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_post_send (3), -.BR ibv_post_recv (3), -.BR ibv_poll_cq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_device.3 b/man/ibv_query_device.3 deleted file mode 100644 index afc7573..0000000 --- a/man/ibv_query_device.3 +++ /dev/null @@ -1,84 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_DEVICE 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_device \- query an RDMA device's attributes -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_device(struct ibv_context " "*context", -.BI " struct ibv_device_attr " "*device_attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_device() -returns the attributes of the device with context -.I context\fR. -The argument -.I device_attr -is a pointer to an ibv_device_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_device_attr { -.in +8 -char fw_ver[64]; /* FW version */ -uint64_t node_guid; /* Node GUID (in network byte order) */ -uint64_t sys_image_guid; /* System image GUID (in network byte order) */ -uint64_t max_mr_size; /* Largest contiguous block that can be registered */ -uint64_t page_size_cap; /* Supported memory shift sizes */ -uint32_t vendor_id; /* Vendor ID, per IEEE */ -uint32_t vendor_part_id; /* Vendor supplied part ID */ -uint32_t hw_ver; /* Hardware version */ -int max_qp; /* Maximum number of supported QPs */ -int max_qp_wr; /* Maximum number of outstanding WR on any work queue */ -int device_cap_flags; /* HCA capabilities mask */ -int max_sge; /* Maximum number of s/g per WR for non-RD QPs */ -int max_sge_rd; /* Maximum number of s/g per WR for RD QPs */ -int max_cq; /* Maximum number of supported CQs */ -int max_cqe; /* Maximum number of CQE capacity per CQ */ -int max_mr; /* Maximum number of supported MRs */ -int max_pd; /* Maximum number of supported PDs */ -int max_qp_rd_atom; /* Maximum number of RDMA Read & Atomic operations that can be outstanding per QP */ -int max_ee_rd_atom; /* Maximum number of RDMA Read & Atomic operations that can be outstanding per EEC */ -int max_res_rd_atom; /* Maximum number of resources used for RDMA Read & Atomic operations by this HCA as the Target */ -int max_qp_init_rd_atom; /* Maximum depth per QP for initiation of RDMA Read & Atomic operations */ -int max_ee_init_rd_atom; /* Maximum depth per EEC for initiation of RDMA Read & Atomic operations */ -enum ibv_atomic_cap atomic_cap; /* Atomic operations support level */ -int max_ee; /* Maximum number of supported EE contexts */ -int max_rdd; /* Maximum number of supported RD domains */ -int max_mw; /* Maximum number of supported MWs */ -int max_raw_ipv6_qp; /* Maximum number of supported raw IPv6 datagram QPs */ -int max_raw_ethy_qp; /* Maximum number of supported Ethertype datagram QPs */ -int max_mcast_grp; /* Maximum number of supported multicast groups */ -int max_mcast_qp_attach; /* Maximum number of QPs per multicast group which can be attached */ -int max_total_mcast_qp_attach;/* Maximum number of QPs which can be attached to multicast groups */ -int max_ah; /* Maximum number of supported address handles */ -int max_fmr; /* Maximum number of supported FMRs */ -int max_map_per_fmr; /* Maximum number of (re)maps per FMR before an unmap operation in required */ -int max_srq; /* Maximum number of supported SRQs */ -int max_srq_wr; /* Maximum number of WRs per SRQ */ -int max_srq_sge; /* Maximum number of s/g per SRQ */ -uint16_t max_pkeys; /* Maximum number of partitions */ -uint8_t local_ca_ack_delay; /* Local CA ack delay */ -uint8_t phys_port_cnt; /* Number of physical ports */ -.in -8 -}; -.fi -.SH "RETURN VALUE" -.B ibv_query_device() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The maximum values returned by this function are the upper limits of -supported resources by the device. However, it may not be possible to -use these maximum values, since the actual number of any resource that -can be created may be limited by the machine configuration, the amount -of host memory, user permissions, and the amount of resources already -in use by other users/processes. -.SH "SEE ALSO" -.BR ibv_open_device (3), -.BR ibv_query_port (3), -.BR ibv_query_pkey (3), -.BR ibv_query_gid (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_device_ex.3 b/man/ibv_query_device_ex.3 deleted file mode 100644 index 88c8ce4..0000000 --- a/man/ibv_query_device_ex.3 +++ /dev/null @@ -1,89 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_DEVICE_EX 3 2014-12-17 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_device_ex \- query an RDMA device's attributes -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_device_ex(struct ibv_context " "*context", -.BI " struct ibv_device_attr_ex " "*attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_device_ex() -returns the attributes of the device with context -.I context\fR. -The argument -.I attr -is a pointer to an ibv_device_attr_ex struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_device_attr_ex { -.in +8 -struct ibv_device_attr orig_attr; -uint32_t comp_mask; /* Compatibility mask that defines which of the following variables are valid */ -struct ibv_odp_caps odp_caps; /* On-Demand Paging capabilities */ -uint64_t completion_timestamp_mask; /* Completion timestamp mask (0 = unsupported) */ -uint64_t hca_core_clock; /* The frequency (in kHZ) of the HCA (0 = unsupported) */ -uint64_t device_cap_flags_ex; /* Extended device capability flags */ -struct ibv_tso_caps tso_caps; /* TCP segmentation offload capabilities */ -struct ibv_rss_caps rss_caps; /* RSS capabilities */ -uint32_t max_wq_type_rq; /* Max Work Queue from type RQ */ -.in -8 -}; - -struct ibv_odp_caps { - uint64_t general_odp_caps; /* Mask with enum ibv_odp_general_cap_bits */ - struct { - uint32_t rc_odp_caps; /* Mask with enum ibv_odp_tranport_cap_bits to know which operations are supported. */ - uint32_t uc_odp_caps; /* Mask with enum ibv_odp_tranport_cap_bits to know which operations are supported. */ - uint32_t ud_odp_caps; /* Mask with enum ibv_odp_tranport_cap_bits to know which operations are supported. */ - } per_transport_caps; -}; - -enum ibv_odp_general_cap_bits { - IBV_ODP_SUPPORT = 1 << 0, /* On demand paging is supported */ -}; - -enum ibv_odp_transport_cap_bits { - IBV_ODP_SUPPORT_SEND = 1 << 0, /* Send operations support on-demand paging */ - IBV_ODP_SUPPORT_RECV = 1 << 1, /* Receive operations support on-demand paging */ - IBV_ODP_SUPPORT_WRITE = 1 << 2, /* RDMA-Write operations support on-demand paging */ - IBV_ODP_SUPPORT_READ = 1 << 3, /* RDMA-Read operations support on-demand paging */ - IBV_ODP_SUPPORT_ATOMIC = 1 << 4, /* RDMA-Atomic operations support on-demand paging */ -}; - -struct ibv_tso_caps { - uint32_t max_tso; /* Maximum payload size in bytes supported for segmentation by TSO engine.*/ - uint32_t supported_qpts; /* Bitmap showing which QP types are supported by TSO operation. */ -}; - -struct ibv_rss_caps { - uint32_t supported_qpts; /* Bitmap showing which QP types are supported RSS */ - uint32_t max_rwq_indirection_tables; /* Max receive work queue indirection tables */ - uint32_t max_rwq_indirection_table_size; /* Max receive work queue indirection table size */ - uint64_t rx_hash_fields_mask; /* Mask with enum ibv_rx_hash_fields to know which incoming packet's field can participates in the RX hash */ - uint8_t rx_hash_function; /* Mask with enum ibv_rx_hash_function_flags to know which hash functions are supported */ -}; - -.fi -.SH "RETURN VALUE" -.B ibv_query_device_ex() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The maximum values returned by this function are the upper limits of -supported resources by the device. However, it may not be possible to -use these maximum values, since the actual number of any resource that -can be created may be limited by the machine configuration, the amount -of host memory, user permissions, and the amount of resources already -in use by other users/processes. -.SH "SEE ALSO" -.BR ibv_query_device (3), -.BR ibv_open_device (3), -.BR ibv_query_port (3), -.BR ibv_query_pkey (3), -.BR ibv_query_gid (3) -.SH "AUTHORS" -.TP -Majd Dibbiny <majd@mellanox.com> diff --git a/man/ibv_query_gid.3 b/man/ibv_query_gid.3 deleted file mode 100644 index b3a1b0f..0000000 --- a/man/ibv_query_gid.3 +++ /dev/null @@ -1,33 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_GID 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_gid \- query an InfiniBand port's GID table -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_gid(struct ibv_context " "*context" ", uint8_t " "port_num" , -.BI " int " "index" ", union ibv_gid " "*gid" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_gid() -returns the GID value in entry -.I index -of port -.I port_num -for device context -.I context -through the pointer -.I gid\fR. -.SH "RETURN VALUE" -.B ibv_query_gid() -returns 0 on success, and \-1 on error. -.SH "SEE ALSO" -.BR ibv_open_device (3), -.BR ibv_query_device (3), -.BR ibv_query_port (3), -.BR ibv_query_pkey (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_pkey.3 b/man/ibv_query_pkey.3 deleted file mode 100644 index 9683049..0000000 --- a/man/ibv_query_pkey.3 +++ /dev/null @@ -1,33 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_PKEY 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_pkey \- query an InfiniBand port's P_Key table -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_pkey(struct ibv_context " "*context" ", uint8_t " "port_num" , -.BI " int " "index" ", uint16_t " "*pkey" "); -.fi -.SH "DESCRIPTION" -.B ibv_query_pkey() -returns the P_Key value (in network byte order) in entry -.I index -of port -.I port_num -for device context -.I context -through the pointer -.I pkey\fR. -.SH "RETURN VALUE" -.B ibv_query_pkey() -returns 0 on success, and \-1 on error. -.SH "SEE ALSO" -.BR ibv_open_device (3), -.BR ibv_query_device (3), -.BR ibv_query_port (3), -.BR ibv_query_gid (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_port.3 b/man/ibv_query_port.3 deleted file mode 100644 index 9bedd90..0000000 --- a/man/ibv_query_port.3 +++ /dev/null @@ -1,65 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_PORT 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_port \- query an RDMA port's attributes -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_port(struct ibv_context " "*context" ", uint8_t " "port_num" , -.BI " struct ibv_port_attr " "*port_attr" "); -.fi -.SH "DESCRIPTION" -.B ibv_query_port() -returns the attributes of port -.I port_num -for device context -.I context -through the pointer -.I port_attr\fR. -The argument -.I port_attr -is an ibv_port_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_port_attr { -.in +8 -enum ibv_port_state state; /* Logical port state */ -enum ibv_mtu max_mtu; /* Max MTU supported by port */ -enum ibv_mtu active_mtu; /* Actual MTU */ -int gid_tbl_len; /* Length of source GID table */ -uint32_t port_cap_flags; /* Port capabilities */ -uint32_t max_msg_sz; /* Maximum message size */ -uint32_t bad_pkey_cntr; /* Bad P_Key counter */ -uint32_t qkey_viol_cntr; /* Q_Key violation counter */ -uint16_t pkey_tbl_len; /* Length of partition table */ -uint16_t lid; /* Base port LID */ -uint16_t sm_lid; /* SM LID */ -uint8_t lmc; /* LMC of LID */ -uint8_t max_vl_num; /* Maximum number of VLs */ -uint8_t sm_sl; /* SM service level */ -uint8_t subnet_timeout; /* Subnet propagation delay */ -uint8_t init_type_reply;/* Type of initialization performed by SM */ -uint8_t active_width; /* Currently active link width */ -uint8_t active_speed; /* Currently active link speed */ -uint8_t phys_state; /* Physical port state */ -uint8_t link_layer; /* link layer protocol of the port */ -.in -8 -}; -.sp -possible values for the link layer field are IBV_LINK_LAYER_INFINIBAND, -IBV_LINK_LAYER_ETHERNET, or IBV_LINK_LAYER_UNSPECIFIED. -.sp -.fi -.SH "RETURN VALUE" -.B ibv_query_port() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_destroy_qp (3), -.BR ibv_query_qp (3), -.BR ibv_create_ah (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_qp.3 b/man/ibv_query_qp.3 deleted file mode 100644 index 3893ec8..0000000 --- a/man/ibv_query_qp.3 +++ /dev/null @@ -1,90 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_QP 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_qp \- get the attributes of a queue pair (QP) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" , -.BI " int " "attr_mask" , -.BI " struct ibv_qp_init_attr " "*init_attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_qp() -gets the attributes specified in -.I attr_mask -for the QP -.I qp -and returns them through the pointers -.I attr -and -.I init_attr\fR. -The argument -.I attr -is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_qp_attr { -.in +8 -enum ibv_qp_state qp_state; /* Current QP state */ -enum ibv_qp_state cur_qp_state; /* Current QP state - irrelevant for ibv_query_qp */ -enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */ -enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */ -uint32_t qkey; /* Q_Key of the QP (valid only for UD QPs) */ -uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */ -uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */ -uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */ -int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */ -struct ibv_qp_cap cap; /* QP capabilities */ -struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */ -struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */ -uint16_t pkey_index; /* Primary P_Key index */ -uint16_t alt_pkey_index; /* Alternate P_Key index */ -uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification - irrelevant for ibv_query_qp */ -uint8_t sq_draining; /* Is the QP draining? (Valid only if qp_state is SQD) */ -uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */ -uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */ -uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */ -uint8_t port_num; /* Primary port number */ -uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */ -uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */ -uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */ -uint8_t alt_port_num; /* Alternate port number */ -uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */ -.in -8 -}; -.fi -.PP -For details on struct ibv_qp_cap see the description of -.B ibv_create_qp()\fR. -For details on struct ibv_ah_attr see the description of -.B ibv_create_ah()\fR. -.SH "RETURN VALUE" -.B ibv_query_qp() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The argument -.I attr_mask -is a hint that specifies the minimum list of attributes to retrieve. -Some RDMA devices may return extra attributes not requested, for -example if the value can be returned cheaply. This has the same -form as in -.B ibv_modify_qp()\fR. -.PP -Attribute values are valid if they have been set using -.B ibv_modify_qp()\fR. -The exact list of valid attributes depends on the QP state. -.PP -Multiple calls to -.B ibv_query_qp() -may yield some differences in the values returned for the following attributes: qp_state, path_mig_state, sq_draining, ah_attr (if APM is enabled). -.SH "SEE ALSO" -.BR ibv_create_qp (3), -.BR ibv_destroy_qp (3), -.BR ibv_modify_qp (3), -.BR ibv_create_ah (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_query_rt_values_ex.3 b/man/ibv_query_rt_values_ex.3 deleted file mode 100644 index fcc460c..0000000 --- a/man/ibv_query_rt_values_ex.3 +++ /dev/null @@ -1,50 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_RT_VALUES_EX 3 2016-2-20 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_rt_values_ex \- query an RDMA device for some real time values -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_rt_values_ex(struct ibv_context " "*context", -.BI " struct ibv_values_ex " "*values" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_rt_values_ex() -returns certain real time values of a device -.I context\fR. -The argument -.I attr -is a pointer to an ibv_device_attr_ex struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_values_ex { -.in +8 -uint32_t comp_mask; /* Compatibility mask that defines the query/queried fields [in/out] */ -struct timespec raw_clock; /* HW raw clock */ -.in -8 -}; - -enum ibv_values_mask { - IBV_VALUES_MASK_RAW_CLOCK = 1 << 0, /* HW raw clock */ -}; - -.fi -.SH "RETURN VALUE" -.B ibv_query_rt_values_ex() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -This extension verb only calls the provider, the provider has to query this value somehow and mark -the queried values in the comp_mask field. -.SH "SEE ALSO" -.BR ibv_query_device (3), -.BR ibv_open_device (3), -.BR ibv_query_port (3), -.BR ibv_query_pkey (3), -.BR ibv_query_gid (3) -.SH "AUTHORS" -.TP -Matan Barak <matanb@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_query_srq.3 b/man/ibv_query_srq.3 deleted file mode 100644 index 05519aa..0000000 --- a/man/ibv_query_srq.3 +++ /dev/null @@ -1,44 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_QUERY_SRQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_query_srq \- get the attributes of a shared receive queue (SRQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_query_srq(struct ibv_srq " "*srq" ", struct ibv_srq_attr " "*srq_attr" ); -.fi -.SH "DESCRIPTION" -.B ibv_query_srq() -gets the attributes of the SRQ -.I srq -and returns them through the pointer -.I srq_attr\fR. -The argument -.I srq_attr -is an ibv_srq_attr struct, as defined in <infiniband/verbs.h>. -.PP -.nf -struct ibv_srq_attr { -.in +8 -uint32_t max_wr; /* maximum number of outstanding work requests (WRs) in the SRQ */ -uint32_t max_sge; /* maximum number of scatter elements per WR */ -uint32_t srq_limit; /* the limit value of the SRQ */ -.in -8 -}; -.fi -.SH "RETURN VALUE" -.B ibv_query_srq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -If the value returned for srq_limit is 0, then the SRQ limit reached -("low watermark") event is not (or no longer) armed, and no -asynchronous events will be generated until the event is rearmed. -.SH "SEE ALSO" -.BR ibv_create_srq (3), -.BR ibv_destroy_srq (3), -.BR ibv_modify_srq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_rate_to_mbps.3 b/man/ibv_rate_to_mbps.3 deleted file mode 100644 index 089db01..0000000 --- a/man/ibv_rate_to_mbps.3 +++ /dev/null @@ -1,45 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_RATE_TO_MBPS 3 2012-03-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -.nf -ibv_rate_to_mbps \- convert IB rate enumeration to Mbit/sec -.sp -mbps_to_ibv_rate \- convert Mbit/sec to an IB rate enumeration -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_rate_to_mbps(enum ibv_rate " "rate" "); -.sp -.BI "enum ibv_rate mbps_to_ibv_rate(int " "mbps" "); -.fi -.SH "DESCRIPTION" -.B ibv_rate_to_mbps() -converts the IB transmission rate enumeration -.I rate -to a number of Mbit/sec. For example, if -.I rate -is -.BR IBV_RATE_5_GBPS\fR, -the value 5000 will be returned (5 Gbit/sec = 5000 Mbit/sec). -.PP -.B mbps_to_ibv_rate() -converts the number of Mbit/sec -.I mult -to an IB transmission rate enumeration. For example, if -.I mult -is 5000, the rate enumeration -.BR IBV_RATE_5_GBPS -will be returned. -.SH "RETURN VALUE" -.B ibv_rate_to_mbps() -returns the number of Mbit/sec. -.PP -.B mbps_to_ibv_rate() -returns the enumeration representing the IB transmission rate. -.SH "SEE ALSO" -.BR ibv_query_port (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanb@dev.mellanox.co.il> diff --git a/man/ibv_rate_to_mult.3 b/man/ibv_rate_to_mult.3 deleted file mode 100644 index fb57ff8..0000000 --- a/man/ibv_rate_to_mult.3 +++ /dev/null @@ -1,46 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_RATE_TO_MULT 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -.nf -ibv_rate_to_mult \- convert IB rate enumeration to multiplier of 2.5 Gbit/sec -.sp -mult_to_ibv_rate \- convert multiplier of 2.5 Gbit/sec to an IB rate enumeration -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_rate_to_mult(enum ibv_rate " "rate" "); -.sp -.BI "enum ibv_rate mult_to_ibv_rate(int " "mult" "); -.fi -.SH "DESCRIPTION" -.B ibv_rate_to_mult() -converts the IB transmission rate enumeration -.I rate -to a multiple of 2.5 Gbit/sec (the base rate). For example, if -.I rate -is -.BR IBV_RATE_5_GBPS\fR, -the value 2 will be returned (5 Gbit/sec = 2 * 2.5 Gbit/sec). -.PP -.B mult_to_ibv_rate() -converts the multiplier value (of 2.5 Gbit/sec) -.I mult -to an IB transmission rate enumeration. For example, if -.I mult -is 2, the rate enumeration -.BR IBV_RATE_5_GBPS -will be returned. -.SH "RETURN VALUE" -.B -ibv_rate_to_mult() -returns the multiplier of the base rate 2.5 Gbit/sec. -.PP -.B mult_to_ibv_rate() -returns the enumeration representing the IB transmission rate. -.SH "SEE ALSO" -.BR ibv_query_port (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_rc_pingpong.1 b/man/ibv_rc_pingpong.1 deleted file mode 100644 index d213c6f..0000000 --- a/man/ibv_rc_pingpong.1 +++ /dev/null @@ -1,64 +0,0 @@ -.TH IBV_RC_PINGPONG 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_rc_pingpong \- simple InfiniBand RC transport test - -.SH SYNOPSIS -.B ibv_rc_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] \fBHOSTNAME\fR - -.B ibv_rc_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] - -.SH DESCRIPTION -.PP -Run a simple ping-pong test over InfiniBand via the reliable -connected (RC) transport. - -.SH OPTIONS - -.PP -.TP -\fB\-p\fR, \fB\-\-port\fR=\fIPORT\fR -use TCP port \fIPORT\fR for initial synchronization (default 18515) -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -use IB port \fIPORT\fR (default port 1) -.TP -\fB\-s\fR, \fB\-\-size\fR=\fISIZE\fR -ping-pong messages of size \fISIZE\fR (default 4096) -.TP -\fB\-r\fR, \fB\-\-rx\-depth\fR=\fIDEPTH\fR -post \fIDEPTH\fR receives at a time (default 1000) -.TP -\fB\-n\fR, \fB\-\-iters\fR=\fIITERS\fR -perform \fIITERS\fR message exchanges (default 1000) -.TP -\fB\-l\fR, \fB\-\-sl\fR=\fISL\fR -use \fISL\fR as the service level value of the QP (default 0) -.TP -\fB\-e\fR, \fB\-\-events\fR -sleep while waiting for work completion events (default is to poll for -completions) - -.SH SEE ALSO -.BR ibv_uc_pingpong (1), -.BR ibv_ud_pingpong (1), -.BR ibv_srq_pingpong (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > - -.SH BUGS -The network synchronization between client and server instances is -weak, and does not prevent incompatible options from being used on the -two instances. The method used for retrieving work completions is not -strictly correct, and race conditions may cause failures on some -systems. diff --git a/man/ibv_reg_mr.3 b/man/ibv_reg_mr.3 deleted file mode 100644 index cf15111..0000000 --- a/man/ibv_reg_mr.3 +++ /dev/null @@ -1,78 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_REG_MR 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_reg_mr, ibv_dereg_mr \- register or deregister a memory region (MR) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "struct ibv_mr *ibv_reg_mr(struct ibv_pd " "*pd" ", void " "*addr" , -.BI " size_t " "length" ", int " "access" ); -.sp -.BI "int ibv_dereg_mr(struct ibv_mr " "*mr" ); -.fi -.SH "DESCRIPTION" -.B ibv_reg_mr() -registers a memory region (MR) associated with the protection domain -.I pd\fR. -The MR's starting address is -.I addr -and its size is -.I length\fR. -The argument -.I access -describes the desired memory protection attributes; it is either 0 or the bitwise OR of one or more of the following flags: -.PP -.TP -.B IBV_ACCESS_LOCAL_WRITE \fR Enable Local Write Access -.TP -.B IBV_ACCESS_REMOTE_WRITE \fR Enable Remote Write Access -.TP -.B IBV_ACCESS_REMOTE_READ\fR Enable Remote Read Access -.TP -.B IBV_ACCESS_REMOTE_ATOMIC\fR Enable Remote Atomic Operation Access (if supported) -.TP -.B IBV_ACCESS_MW_BIND\fR Enable Memory Window Binding -.TP -.B IBV_ACCESS_ON_DEMAND\fR Create an on-demand paging MR -.PP -If -.B IBV_ACCESS_REMOTE_WRITE -or -.B IBV_ACCESS_REMOTE_ATOMIC -is set, then -.B IBV_ACCESS_LOCAL_WRITE -must be set too. -.PP -Local read access is always enabled for the MR. -.PP -.B ibv_dereg_mr() -deregisters the MR -.I mr\fR. -.SH "RETURN VALUE" -.B ibv_reg_mr() -returns a pointer to the registered MR, or NULL if the request fails. -The local key (\fBL_Key\fR) field -.B lkey -is used as the lkey field of struct ibv_sge when posting buffers with -ibv_post_* verbs, and the the remote key (\fBR_Key\fR) -field -.B rkey -is used by remote processes to perform Atomic and RDMA operations. The remote process places this -.B rkey -as the rkey field of struct ibv_send_wr passed to the ibv_post_send function. -.PP -.B ibv_dereg_mr() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_dereg_mr() -fails if any memory window is still bound to this MR. -.SH "SEE ALSO" -.BR ibv_alloc_pd (3), -.BR ibv_post_send (3), -.BR ibv_post_recv (3), -.BR ibv_post_srq_recv (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_req_notify_cq.3 b/man/ibv_req_notify_cq.3 deleted file mode 100644 index 47d45fe..0000000 --- a/man/ibv_req_notify_cq.3 +++ /dev/null @@ -1,43 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_REQ_NOTIFY_CQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_req_notify_cq \- request completion notification on a completion queue (CQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_req_notify_cq(struct ibv_cq " "*cq" ", int " "solicited_only" "); -.SH "DESCRIPTION" -.B ibv_req_notify_cq() -requests a completion notification on the completion queue (CQ) -.I cq\fR. -.PP -Upon the addition of a new CQ entry (CQE) to -.I cq\fR, -a completion event will be added to the completion channel associated -with the CQ. -If the argument -.I solicited_only -is zero, a completion event is generated for any new CQE. If -.I solicited_only -is non\-zero, an event is only generated for a new CQE with that is -considered "solicited." A CQE is solicited if it is a receive -completion for a message with the Solicited Event header bit set, or -if the status is not successful. All other successful receive -completions, or any successful send completion is unsolicited. -.SH "RETURN VALUE" -.B -ibv_req_notify_cq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -The request for notification is "one shot." Only one completion event -will be generated for each call to -.B ibv_req_notify_cq()\fR. -.SH "SEE ALSO" -.BR ibv_create_comp_channel (3), -.BR ibv_create_cq (3), -.BR ibv_get_cq_event (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_rereg_mr.3 b/man/ibv_rereg_mr.3 deleted file mode 100644 index a678437..0000000 --- a/man/ibv_rereg_mr.3 +++ /dev/null @@ -1,76 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_REREG_MR 3 2016-03-13 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_rereg_mr \- re-register a memory region (MR) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_rereg_mr(struct ibv_mr " "*mr" ", int " " flags" , -.BI " struct ibv_pd * " "pd" ", void " " *addr", -.BI " size_t " " length" ", int " " access"); -.fi -.fi -.SH "DESCRIPTION" -.B ibv_rereg_mr() -Modifies the attributes of an existing memory region (MR) -.I mr\fR. -Conceptually, this call performs the functions deregister memory region -followed by register memory region. Where possible, -resources are reused instead of deallocated and reallocated. -.PP -.I flags\fR -is a bit-mask used to indicate which of the following properties of the memory region are being modified. Flags should be a combination (bit field) of: -.PP -.TP -.B IBV_REREG_MR_CHANGE_TRANSLATION \fR Change translation (location and length) -.TP -.B IBV_REREG_MR_CHANGE_PD \fR Change protection domain -.TP -.B IBV_REREG_MR_CHANGE_ACCESS \fR Change access flags -.PP -When -.B IBV_REREG_MR_CHANGE_PD -is used, -.I pd\fR -represents the new PD this MR should be registered to. -.br -When -.B IBV_REREG_MR_CHANGE_TRANSLATION -is used, -.I addr\fR. -represents the virtual address (user-space pointer) of the new MR, while -.I length\fR -represents its length. -.PP -The access and other flags are represented in the field -.I access\fR. -This field describes the desired memory protection attributes; it is either 0 or the bitwise OR of one or more of ibv_access_flags. -.TP -.SH "RETURN VALUE" -.B ibv_rereg_mr() -returns 0 on success, otherwise an error has occurred, -.I enum ibv_rereg_mr_err_code\fR -represents the error as of below. -.br -IBV_REREG_MR_ERR_INPUT - Old MR is valid, an input error was detected by libibverbs. -.br -IBV_REREG_MR_ERR_DONT_FORK_NEW - Old MR is valid, failed via dont fork on new address range. -.br -IBV_REREG_MR_ERR_DO_FORK_OLD - New MR is valid, failed via do fork on old address range. -.br -IBV_REREG_MR_ERR_CMD - MR shouldn't be used, command error. -.br -IBV_REREG_MR_ERR_CMD_AND_DO_FORK_NEW - MR shouldn't be used, command error, invalid fork state on new address range. - -.SH "NOTES" -Even on a failure, the user still needs to call ibv_dereg_mr on this MR. -.SH "SEE ALSO" -.BR ibv_reg_mr (3), -.BR ibv_dereg_mr (3), -.SH "AUTHORS" -.TP -Matan Barak <matanb@mellanox.com> -.TP -Yishai Hadas <yishaih@mellanox.com> diff --git a/man/ibv_resize_cq.3 b/man/ibv_resize_cq.3 deleted file mode 100644 index fbb1ebd..0000000 --- a/man/ibv_resize_cq.3 +++ /dev/null @@ -1,42 +0,0 @@ -.\" -*- nroff -*- -.\" -.TH IBV_RESIZE_CQ 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual" -.SH "NAME" -ibv_resize_cq \- resize a completion queue (CQ) -.SH "SYNOPSIS" -.nf -.B #include <infiniband/verbs.h> -.sp -.BI "int ibv_resize_cq(struct ibv_cq " "*cq" ", int " "cqe" "); -.fi -.SH "DESCRIPTION" -.B ibv_resize_cq() -resizes the completion queue (CQ) -.I cq -to have at least -.I cqe -entries. -.I cqe -must be at least the number of unpolled entries in the CQ -.I cq\fR. -If -.I cqe -is a valid value less than the current CQ size, -.B ibv_resize_cq() -may not do anything, since this function is only guaranteed to resize -the CQ to a size at least as big as the requested size. -.SH "RETURN VALUE" -.B ibv_resize_cq() -returns 0 on success, or the value of errno on failure (which indicates the failure reason). -.SH "NOTES" -.B ibv_resize_cq() -may assign a CQ size greater than or equal to the requested size. -The cqe member of -.I cq -will be updated to the actual size. -.SH "SEE ALSO" -.BR ibv_create_cq (3) -.BR ibv_destroy_cq (3) -.SH "AUTHORS" -.TP -Dotan Barak <dotanba@gmail.com> diff --git a/man/ibv_srq_pingpong.1 b/man/ibv_srq_pingpong.1 deleted file mode 100644 index d50f70e..0000000 --- a/man/ibv_srq_pingpong.1 +++ /dev/null @@ -1,68 +0,0 @@ -.TH IBV_SRQ_PINGPONG 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_srq_pingpong \- simple InfiniBand shared receive queue test - -.SH SYNOPSIS -.B ibv_srq_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-q num QPs] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] \fBHOSTNAME\fR - -.B ibv_srq_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-q num QPs] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] - -.SH DESCRIPTION -.PP -Run a simple ping-pong test over InfiniBand via the reliable -connected (RC) transport, using multiple queue pairs (QPs) and a -single shared receive queue (SRQ). - -.SH OPTIONS - -.PP -.TP -\fB\-p\fR, \fB\-\-port\fR=\fIPORT\fR -use TCP port \fIPORT\fR for initial synchronization (default 18515) -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -use IB port \fIPORT\fR (default port 1) -.TP -\fB\-s\fR, \fB\-\-size\fR=\fISIZE\fR -ping-pong messages of size \fISIZE\fR (default 4096) -.TP -\fB\-q\fR, \fB\-\-num\-qp\fR=\fINUM\fR -use \fINUM\fR queue pairs for test (default 16) -.TP -\fB\-r\fR, \fB\-\-rx\-depth\fR=\fIDEPTH\fR -post \fIDEPTH\fR receives at a time (default 1000) -.TP -\fB\-n\fR, \fB\-\-iters\fR=\fIITERS\fR -perform \fIITERS\fR message exchanges (default 1000) -.TP -\fB\-l\fR, \fB\-\-sl\fR=\fISL\fR -use \fISL\fR as the service level value of the QPs (default 0) -.TP -\fB\-e\fR, \fB\-\-events\fR -sleep while waiting for work completion events (default is to poll for -completions) - -.SH SEE ALSO -.BR ibv_rc_pingpong (1), -.BR ibv_uc_pingpong (1), -.BR ibv_ud_pingpong (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > - -.SH BUGS -The network synchronization between client and server instances is -weak, and does not prevent incompatible options from being used on the -two instances. The method used for retrieving work completions is not -strictly correct, and race conditions may cause failures on some -systems. diff --git a/man/ibv_uc_pingpong.1 b/man/ibv_uc_pingpong.1 deleted file mode 100644 index ec97eb0..0000000 --- a/man/ibv_uc_pingpong.1 +++ /dev/null @@ -1,64 +0,0 @@ -.TH IBV_UC_PINGPONG 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_uc_pingpong \- simple InfiniBand UC transport test - -.SH SYNOPSIS -.B ibv_uc_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] \fBHOSTNAME\fR - -.B ibv_uc_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] - -.SH DESCRIPTION -.PP -Run a simple ping-pong test over InfiniBand via the reliable -connected (RC) transport. - -.SH OPTIONS - -.PP -.TP -\fB\-p\fR, \fB\-\-port\fR=\fIPORT\fR -use TCP port \fIPORT\fR for initial synchronization (default 18515) -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -use IB port \fIPORT\fR (default port 1) -.TP -\fB\-s\fR, \fB\-\-size\fR=\fISIZE\fR -ping-pong messages of size \fISIZE\fR (default 4096) -.TP -\fB\-r\fR, \fB\-\-rx\-depth\fR=\fIDEPTH\fR -post \fIDEPTH\fR receives at a time (default 1000) -.TP -\fB\-n\fR, \fB\-\-iters\fR=\fIITERS\fR -perform \fIITERS\fR message exchanges (default 1000) -.TP -\fB\-l\fR, \fB\-\-sl\fR=\fISL\fR -use \fISL\fR as the service level value of the QP (default 0) -.TP -\fB\-e\fR, \fB\-\-events\fR -sleep while waiting for work completion events (default is to poll for -completions) - -.SH SEE ALSO -.BR ibv_rc_pingpong (1), -.BR ibv_ud_pingpong (1), -.BR ibv_srq_pingpong (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > - -.SH BUGS -The network synchronization between client and server instances is -weak, and does not prevent incompatible options from being used on the -two instances. The method used for retrieving work completions is not -strictly correct, and race conditions may cause failures on some -systems. diff --git a/man/ibv_ud_pingpong.1 b/man/ibv_ud_pingpong.1 deleted file mode 100644 index a05af8d..0000000 --- a/man/ibv_ud_pingpong.1 +++ /dev/null @@ -1,64 +0,0 @@ -.TH IBV_UD_PINGPONG 1 "August 30, 2005" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_ud_pingpong \- simple InfiniBand UD transport test - -.SH SYNOPSIS -.B ibv_ud_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] \fBHOSTNAME\fR - -.B ibv_ud_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-r rx depth] -[\-n iters] [\-l sl] [\-e] - -.SH DESCRIPTION -.PP -Run a simple ping-pong test over InfiniBand via the unreliable -datagram (UD) transport. - -.SH OPTIONS - -.PP -.TP -\fB\-p\fR, \fB\-\-port\fR=\fIPORT\fR -use TCP port \fIPORT\fR for initial synchronization (default 18515) -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -use IB port \fIPORT\fR (default port 1) -.TP -\fB\-s\fR, \fB\-\-size\fR=\fISIZE\fR -ping-pong messages of size \fISIZE\fR (default 2048) -.TP -\fB\-r\fR, \fB\-\-rx\-depth\fR=\fIDEPTH\fR -post \fIDEPTH\fR receives at a time (default 500) -.TP -\fB\-n\fR, \fB\-\-iters\fR=\fIITERS\fR -perform \fIITERS\fR message exchanges (default 1000) -.TP -\fB\-l\fR, \fB\-\-sl\fR=\fISL\fR -send messages with service level \fISL\fR (default 0) -.TP -\fB\-e\fR, \fB\-\-events\fR -sleep while waiting for work completion events (default is to poll for -completions) - -.SH SEE ALSO -.BR ibv_rc_pingpong (1), -.BR ibv_uc_pingpong (1), -.BR ibv_srq_pingpong (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < rolandd@cisco.com > - -.SH BUGS -The network synchronization between client and server instances is -weak, and does not prevent incompatible options from being used on the -two instances. The method used for retrieving work completions is not -strictly correct, and race conditions may cause failures on some -systems. diff --git a/man/ibv_xsrq_pingpong.1 b/man/ibv_xsrq_pingpong.1 deleted file mode 100644 index db4a988..0000000 --- a/man/ibv_xsrq_pingpong.1 +++ /dev/null @@ -1,71 +0,0 @@ -.TH IBV_XSRQ_PINGPONG 1 "May 24, 2016" "libibverbs" "USER COMMANDS" - -.SH NAME -ibv_xsrq_pingpong \- simple InfiniBand shared receive queue test - -.SH SYNOPSIS -.B ibv_xsrq_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-m mtu] [\-c clients] -[\-n num_tests] [\-l sl] [\-e] \fBHOSTNAME\fR - -.B ibv_xsrq_pingpong -[\-p port] [\-d device] [\-i ib port] [\-s size] [\-m mtu] [\-c clients] -[\-n num_tests] [\-l sl] [\-e] - -.SH DESCRIPTION -.PP -Run a simple ping-pong test over InfiniBand via the extended reliable -connected (XRC) transport service, using a shared receive queue (SRQ). - -.SH OPTIONS - -.PP -.TP -\fB\-p\fR, \fB\-\-port\fR=\fIPORT\fR -use TCP port \fIPORT\fR for initial synchronization (default 18515) -.TP -\fB\-d\fR, \fB\-\-ib\-dev\fR=\fIDEVICE\fR -use IB device \fIDEVICE\fR (default first device found) -.TP -\fB\-i\fR, \fB\-\-ib\-port\fR=\fIPORT\fR -use IB port \fIPORT\fR (default port 1) -.TP -\fB\-s\fR, \fB\-\-size\fR=\fISIZE\fR -ping-pong messages of size \fISIZE\fR (default 4096) -.TP -\fB\-m\fR, \fB\-\-mtu\fR=\fIMTU\fR -use path mtu of size \fIMTU\fR (default 2048) -.TP -\fB\-c\fR, \fB\-\-clients\fR=\fICLIENTS\fR -number of clients \fICLIENTS\fR (on server only, default 1) -.TP -\fB\-n\fR, \fB\-\-num\-tests\fR=\fINUM_TESTS\fR -perform \fINUM_TESTS\fR tests per client (default 5) -.TP -\fB\-l\fR, \fB\-\-sl\fR=\fISL\fR -use \fISL\fR as the service level value (default 0) -.TP -\fB\-e\fR, \fB\-\-events\fR -sleep while waiting for work completion events (default is to poll for -completions) - -.SH SEE ALSO -.BR ibv_rc_pingpong (1), -.BR ibv_uc_pingpong (1), -.BR ibv_ud_pingpong (1) -.BR ibv_srq_pingpong (1) - -.SH AUTHORS -.TP -Roland Dreier -.RI < roland@purestorage.com > -.TP -Jarod Wilson -.RI < jarod@redhat.com > - -.SH BUGS -The network synchronization between client and server instances is -weak, and does not prevent incompatible options from being used on the -two instances. The method used for retrieving work completions is not -strictly correct, and race conditions may cause failures on some -systems. diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index 7297cbb..0000000 --- a/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.la -.dirstamp -.libs diff --git a/src/cmd.c b/src/cmd.c deleted file mode 100644 index 11f6509..0000000 --- a/src/cmd.c +++ /dev/null @@ -1,1864 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 PathScale, Inc. All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <unistd.h> -#include <stdlib.h> -#include <errno.h> -#include <alloca.h> -#include <string.h> - -#include "ibverbs.h" - - -int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, - size_t cmd_size, struct ibv_get_context_resp *resp, - size_t resp_size) -{ - if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION) - return ENOSYS; - - IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size); - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - context->async_fd = resp->async_fd; - context->num_comp_vectors = resp->num_comp_vectors; - - return 0; -} - -static void copy_query_dev_fields(struct ibv_device_attr *device_attr, - struct ibv_query_device_resp *resp, - uint64_t *raw_fw_ver) -{ - *raw_fw_ver = resp->fw_ver; - device_attr->node_guid = resp->node_guid; - device_attr->sys_image_guid = resp->sys_image_guid; - device_attr->max_mr_size = resp->max_mr_size; - device_attr->page_size_cap = resp->page_size_cap; - device_attr->vendor_id = resp->vendor_id; - device_attr->vendor_part_id = resp->vendor_part_id; - device_attr->hw_ver = resp->hw_ver; - device_attr->max_qp = resp->max_qp; - device_attr->max_qp_wr = resp->max_qp_wr; - device_attr->device_cap_flags = resp->device_cap_flags; - device_attr->max_sge = resp->max_sge; - device_attr->max_sge_rd = resp->max_sge_rd; - device_attr->max_cq = resp->max_cq; - device_attr->max_cqe = resp->max_cqe; - device_attr->max_mr = resp->max_mr; - device_attr->max_pd = resp->max_pd; - device_attr->max_qp_rd_atom = resp->max_qp_rd_atom; - device_attr->max_ee_rd_atom = resp->max_ee_rd_atom; - device_attr->max_res_rd_atom = resp->max_res_rd_atom; - device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom; - device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom; - device_attr->atomic_cap = resp->atomic_cap; - device_attr->max_ee = resp->max_ee; - device_attr->max_rdd = resp->max_rdd; - device_attr->max_mw = resp->max_mw; - device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp; - device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp; - device_attr->max_mcast_grp = resp->max_mcast_grp; - device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach; - device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach; - device_attr->max_ah = resp->max_ah; - device_attr->max_fmr = resp->max_fmr; - device_attr->max_map_per_fmr = resp->max_map_per_fmr; - device_attr->max_srq = resp->max_srq; - device_attr->max_srq_wr = resp->max_srq_wr; - device_attr->max_srq_sge = resp->max_srq_sge; - device_attr->max_pkeys = resp->max_pkeys; - device_attr->local_ca_ack_delay = resp->local_ca_ack_delay; - device_attr->phys_port_cnt = resp->phys_port_cnt; -} - -int ibv_cmd_query_device(struct ibv_context *context, - struct ibv_device_attr *device_attr, - uint64_t *raw_fw_ver, - struct ibv_query_device *cmd, size_t cmd_size) -{ - struct ibv_query_device_resp resp; - - IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_DEVICE, &resp, sizeof resp); - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - memset(device_attr->fw_ver, 0, sizeof device_attr->fw_ver); - copy_query_dev_fields(device_attr, &resp, raw_fw_ver); - - return 0; -} - -int ibv_cmd_query_device_ex(struct ibv_context *context, - const struct ibv_query_device_ex_input *input, - struct ibv_device_attr_ex *attr, size_t attr_size, - uint64_t *raw_fw_ver, - struct ibv_query_device_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_query_device_resp_ex *resp, - size_t resp_core_size, - size_t resp_size) -{ - int err; - - if (input && input->comp_mask) - return EINVAL; - - if (attr_size < offsetof(struct ibv_device_attr_ex, comp_mask) + - sizeof(attr->comp_mask)) - return EINVAL; - - if (resp_core_size < offsetof(struct ibv_query_device_resp_ex, - response_length) + - sizeof(resp->response_length)) - return EINVAL; - - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, - QUERY_DEVICE_EX, resp, resp_core_size, - resp_size); - cmd->comp_mask = 0; - cmd->reserved = 0; - memset(attr->orig_attr.fw_ver, 0, sizeof(attr->orig_attr.fw_ver)); - memset(&attr->comp_mask, 0, attr_size - sizeof(attr->orig_attr)); - err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; - - (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - copy_query_dev_fields(&attr->orig_attr, &resp->base, raw_fw_ver); - /* Report back supported comp_mask bits. For now no comp_mask bit is - * defined */ - attr->comp_mask = resp->comp_mask & 0; - if (attr_size >= offsetof(struct ibv_device_attr_ex, odp_caps) + - sizeof(attr->odp_caps)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, odp_caps) + - sizeof(resp->odp_caps)) { - attr->odp_caps.general_caps = resp->odp_caps.general_caps; - attr->odp_caps.per_transport_caps.rc_odp_caps = - resp->odp_caps.per_transport_caps.rc_odp_caps; - attr->odp_caps.per_transport_caps.uc_odp_caps = - resp->odp_caps.per_transport_caps.uc_odp_caps; - attr->odp_caps.per_transport_caps.ud_odp_caps = - resp->odp_caps.per_transport_caps.ud_odp_caps; - } - } - - if (attr_size >= offsetof(struct ibv_device_attr_ex, - completion_timestamp_mask) + - sizeof(attr->completion_timestamp_mask)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, timestamp_mask) + - sizeof(resp->timestamp_mask)) - attr->completion_timestamp_mask = resp->timestamp_mask; - } - - if (attr_size >= offsetof(struct ibv_device_attr_ex, hca_core_clock) + - sizeof(attr->hca_core_clock)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, hca_core_clock) + - sizeof(resp->hca_core_clock)) - attr->hca_core_clock = resp->hca_core_clock; - } - - if (attr_size >= offsetof(struct ibv_device_attr_ex, device_cap_flags_ex) + - sizeof(attr->device_cap_flags_ex)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, device_cap_flags_ex) + - sizeof(resp->device_cap_flags_ex)) - attr->device_cap_flags_ex = resp->device_cap_flags_ex; - } - - if (attr_size >= offsetof(struct ibv_device_attr_ex, rss_caps) + - sizeof(attr->rss_caps)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, rss_caps) + - sizeof(resp->rss_caps)) { - attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; - attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables; - attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size; - } - } - - if (attr_size >= offsetof(struct ibv_device_attr_ex, max_wq_type_rq) + - sizeof(attr->max_wq_type_rq)) { - if (resp->response_length >= - offsetof(struct ibv_query_device_resp_ex, max_wq_type_rq) + - sizeof(resp->max_wq_type_rq)) - attr->max_wq_type_rq = resp->max_wq_type_rq; - } - - return 0; -} - -int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr, - struct ibv_query_port *cmd, size_t cmd_size) -{ - struct ibv_query_port_resp resp; - - IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT, &resp, sizeof resp); - cmd->port_num = port_num; - memset(cmd->reserved, 0, sizeof cmd->reserved); - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - port_attr->state = resp.state; - port_attr->max_mtu = resp.max_mtu; - port_attr->active_mtu = resp.active_mtu; - port_attr->gid_tbl_len = resp.gid_tbl_len; - port_attr->port_cap_flags = resp.port_cap_flags; - port_attr->max_msg_sz = resp.max_msg_sz; - port_attr->bad_pkey_cntr = resp.bad_pkey_cntr; - port_attr->qkey_viol_cntr = resp.qkey_viol_cntr; - port_attr->pkey_tbl_len = resp.pkey_tbl_len; - port_attr->lid = resp.lid; - port_attr->sm_lid = resp.sm_lid; - port_attr->lmc = resp.lmc; - port_attr->max_vl_num = resp.max_vl_num; - port_attr->sm_sl = resp.sm_sl; - port_attr->subnet_timeout = resp.subnet_timeout; - port_attr->init_type_reply = resp.init_type_reply; - port_attr->active_width = resp.active_width; - port_attr->active_speed = resp.active_speed; - port_attr->phys_state = resp.phys_state; - port_attr->link_layer = resp.link_layer; - - return 0; -} - -int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, - struct ibv_alloc_pd *cmd, size_t cmd_size, - struct ibv_alloc_pd_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_PD, resp, resp_size); - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - pd->handle = resp->pd_handle; - pd->context = context; - - return 0; -} - -int ibv_cmd_dealloc_pd(struct ibv_pd *pd) -{ - struct ibv_dealloc_pd cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, DEALLOC_PD); - cmd.pd_handle = pd->handle; - - if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd, - int vxrcd_size, - struct ibv_xrcd_init_attr *attr, - struct ibv_open_xrcd *cmd, size_t cmd_size, - struct ibv_open_xrcd_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size); - - if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED) - return ENOSYS; - - if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) || - !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS)) - return EINVAL; - - cmd->fd = attr->fd; - cmd->oflags = attr->oflags; - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - xrcd->xrcd.context = context; - xrcd->comp_mask = 0; - if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) { - xrcd->comp_mask = VERBS_XRCD_HANDLE; - xrcd->handle = resp->xrcd_handle; - } - - return 0; -} - -int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd) -{ - struct ibv_close_xrcd cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD); - cmd.xrcd_handle = xrcd->handle; - - if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, int access, - struct ibv_mr *mr, struct ibv_reg_mr *cmd, - size_t cmd_size, - struct ibv_reg_mr_resp *resp, size_t resp_size) -{ - - IBV_INIT_CMD_RESP(cmd, cmd_size, REG_MR, resp, resp_size); - - cmd->start = (uintptr_t) addr; - cmd->length = length; - cmd->hca_va = hca_va; - cmd->pd_handle = pd->handle; - cmd->access_flags = access; - - if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - mr->handle = resp->mr_handle; - mr->lkey = resp->lkey; - mr->rkey = resp->rkey; - mr->context = pd->context; - - return 0; -} - -int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, - size_t length, uint64_t hca_va, int access, - struct ibv_pd *pd, struct ibv_rereg_mr *cmd, - size_t cmd_sz, struct ibv_rereg_mr_resp *resp, - size_t resp_sz) -{ - IBV_INIT_CMD_RESP(cmd, cmd_sz, REREG_MR, resp, resp_sz); - - cmd->mr_handle = mr->handle; - cmd->flags = flags; - cmd->start = (uintptr_t)addr; - cmd->length = length; - cmd->hca_va = hca_va; - cmd->pd_handle = (flags & IBV_REREG_MR_CHANGE_PD) ? pd->handle : 0; - cmd->access_flags = access; - - if (write(mr->context->cmd_fd, cmd, cmd_sz) != cmd_sz) - return errno; - - (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_sz); - - mr->lkey = resp->lkey; - mr->rkey = resp->rkey; - if (flags & IBV_REREG_MR_CHANGE_PD) - mr->context = pd->context; - - return 0; -} - -int ibv_cmd_dereg_mr(struct ibv_mr *mr) -{ - struct ibv_dereg_mr cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, DEREG_MR); - cmd.mr_handle = mr->handle; - - if (write(mr->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type, - struct ibv_mw *mw, struct ibv_alloc_mw *cmd, - size_t cmd_size, - struct ibv_alloc_mw_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_MW, resp, resp_size); - cmd->pd_handle = pd->handle; - cmd->mw_type = type; - memset(cmd->reserved, 0, sizeof(cmd->reserved)); - - if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - mw->context = pd->context; - mw->pd = pd; - mw->rkey = resp->rkey; - mw->handle = resp->mw_handle; - mw->type = type; - - return 0; -} - -int ibv_cmd_dealloc_mw(struct ibv_mw *mw, - struct ibv_dealloc_mw *cmd, size_t cmd_size) -{ - IBV_INIT_CMD(cmd, cmd_size, DEALLOC_MW); - cmd->mw_handle = mw->handle; - cmd->reserved = 0; - - if (write(mw->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - return 0; -} - -int ibv_cmd_create_cq(struct ibv_context *context, int cqe, - struct ibv_comp_channel *channel, - int comp_vector, struct ibv_cq *cq, - struct ibv_create_cq *cmd, size_t cmd_size, - struct ibv_create_cq_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size); - cmd->user_handle = (uintptr_t) cq; - cmd->cqe = cqe; - cmd->comp_vector = comp_vector; - cmd->comp_channel = channel ? channel->fd : -1; - cmd->reserved = 0; - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - cq->handle = resp->cq_handle; - cq->cqe = resp->cqe; - cq->context = context; - - return 0; -} - -int ibv_cmd_create_cq_ex(struct ibv_context *context, - struct ibv_cq_init_attr_ex *cq_attr, - struct ibv_cq_ex *cq, - struct ibv_create_cq_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_cq_resp_ex *resp, - size_t resp_core_size, - size_t resp_size) -{ - int err; - - memset(cmd, 0, cmd_core_size); - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ_EX, resp, - resp_core_size, resp_size); - - if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1)) - return EINVAL; - - cmd->user_handle = (uintptr_t)cq; - cmd->cqe = cq_attr->cqe; - cmd->comp_vector = cq_attr->comp_vector; - cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1; - cmd->comp_mask = 0; - - if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) + - sizeof(cmd->flags)) { - if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) && - (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1))) - return EOPNOTSUPP; - - if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) - cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP; - } - - err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; - - (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - cq->handle = resp->base.cq_handle; - cq->cqe = resp->base.cqe; - cq->context = context; - - return 0; -} - -int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) -{ - struct ibv_poll_cq cmd; - struct ibv_poll_cq_resp *resp; - int i; - int rsize; - int ret; - - rsize = sizeof *resp + ne * sizeof(struct ibv_kern_wc); - resp = malloc(rsize); - if (!resp) - return -1; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, POLL_CQ, resp, rsize); - cmd.cq_handle = ibcq->handle; - cmd.ne = ne; - - if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) { - ret = -1; - goto out; - } - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, rsize); - - for (i = 0; i < resp->count; i++) { - wc[i].wr_id = resp->wc[i].wr_id; - wc[i].status = resp->wc[i].status; - wc[i].opcode = resp->wc[i].opcode; - wc[i].vendor_err = resp->wc[i].vendor_err; - wc[i].byte_len = resp->wc[i].byte_len; - wc[i].imm_data = resp->wc[i].imm_data; - wc[i].qp_num = resp->wc[i].qp_num; - wc[i].src_qp = resp->wc[i].src_qp; - wc[i].wc_flags = resp->wc[i].wc_flags; - wc[i].pkey_index = resp->wc[i].pkey_index; - wc[i].slid = resp->wc[i].slid; - wc[i].sl = resp->wc[i].sl; - wc[i].dlid_path_bits = resp->wc[i].dlid_path_bits; - } - - ret = resp->count; - -out: - free(resp); - return ret; -} - -int ibv_cmd_req_notify_cq(struct ibv_cq *ibcq, int solicited_only) -{ - struct ibv_req_notify_cq cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, REQ_NOTIFY_CQ); - cmd.cq_handle = ibcq->handle; - cmd.solicited = !!solicited_only; - - if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe, - struct ibv_resize_cq *cmd, size_t cmd_size, - struct ibv_resize_cq_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size); - cmd->cq_handle = cq->handle; - cmd->cqe = cqe; - - if (write(cq->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - cq->cqe = resp->cqe; - - return 0; -} - -int ibv_cmd_destroy_cq(struct ibv_cq *cq) -{ - struct ibv_destroy_cq cmd; - struct ibv_destroy_cq_resp resp; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp); - cmd.cq_handle = cq->handle; - cmd.reserved = 0; - - if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - pthread_mutex_lock(&cq->mutex); - while (cq->comp_events_completed != resp.comp_events_reported || - cq->async_events_completed != resp.async_events_reported) - pthread_cond_wait(&cq->cond, &cq->mutex); - pthread_mutex_unlock(&cq->mutex); - - return 0; -} - -int ibv_cmd_create_srq(struct ibv_pd *pd, - struct ibv_srq *srq, struct ibv_srq_init_attr *attr, - struct ibv_create_srq *cmd, size_t cmd_size, - struct ibv_create_srq_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_SRQ, resp, resp_size); - cmd->user_handle = (uintptr_t) srq; - cmd->pd_handle = pd->handle; - cmd->max_wr = attr->attr.max_wr; - cmd->max_sge = attr->attr.max_sge; - cmd->srq_limit = attr->attr.srq_limit; - - if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - srq->handle = resp->srq_handle; - srq->context = pd->context; - - if (abi_ver > 5) { - attr->attr.max_wr = resp->max_wr; - attr->attr.max_sge = resp->max_sge; - } else { - struct ibv_create_srq_resp_v5 *resp_v5 = - (struct ibv_create_srq_resp_v5 *) resp; - - memmove((void *) resp + sizeof *resp, - (void *) resp_v5 + sizeof *resp_v5, - resp_size - sizeof *resp); - } - - return 0; -} - -int ibv_cmd_create_srq_ex(struct ibv_context *context, - struct verbs_srq *srq, int vsrq_sz, - struct ibv_srq_init_attr_ex *attr_ex, - struct ibv_create_xsrq *cmd, size_t cmd_size, - struct ibv_create_srq_resp *resp, size_t resp_size) -{ - struct verbs_xrcd *vxrcd = NULL; - - IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size); - - if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED) - return ENOSYS; - - if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD)) - return EINVAL; - - cmd->user_handle = (uintptr_t) srq; - cmd->pd_handle = attr_ex->pd->handle; - cmd->max_wr = attr_ex->attr.max_wr; - cmd->max_sge = attr_ex->attr.max_sge; - cmd->srq_limit = attr_ex->attr.srq_limit; - - cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? - attr_ex->srq_type : IBV_SRQT_BASIC; - if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { - if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ)) - return EINVAL; - - vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd); - cmd->xrcd_handle = vxrcd->handle; - cmd->cq_handle = attr_ex->cq->handle; - } - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - srq->srq.handle = resp->srq_handle; - srq->srq.context = context; - srq->srq.srq_context = attr_ex->srq_context; - srq->srq.pd = attr_ex->pd; - srq->srq.events_completed = 0; - pthread_mutex_init(&srq->srq.mutex, NULL); - pthread_cond_init(&srq->srq.cond, NULL); - - /* - * check that the last field is available. - * If it is than all the others exist as well - */ - if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) { - srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE; - srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? - attr_ex->srq_type : IBV_SRQT_BASIC; - if (srq->srq_type == IBV_SRQT_XRC) { - srq->comp_mask |= VERBS_SRQ_NUM; - srq->srq_num = resp->srqn; - } - if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { - srq->comp_mask |= VERBS_SRQ_XRCD; - srq->xrcd = vxrcd; - } - if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) { - srq->comp_mask |= VERBS_SRQ_CQ; - srq->cq = attr_ex->cq; - } - } - - attr_ex->attr.max_wr = resp->max_wr; - attr_ex->attr.max_sge = resp->max_sge; - - return 0; -} - - -static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask, - struct ibv_modify_srq *new_cmd, - size_t new_cmd_size) -{ - struct ibv_modify_srq_v3 *cmd; - size_t cmd_size; - - cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd; - cmd = alloca(cmd_size); - memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd); - - IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); - - cmd->srq_handle = srq->handle; - cmd->attr_mask = srq_attr_mask; - cmd->max_wr = srq_attr->max_wr; - cmd->srq_limit = srq_attr->srq_limit; - cmd->max_sge = 0; - cmd->reserved = 0; - - if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - return 0; -} - -int ibv_cmd_modify_srq(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask, - struct ibv_modify_srq *cmd, size_t cmd_size) -{ - if (abi_ver == 3) - return ibv_cmd_modify_srq_v3(srq, srq_attr, srq_attr_mask, - cmd, cmd_size); - - IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); - - cmd->srq_handle = srq->handle; - cmd->attr_mask = srq_attr_mask; - cmd->max_wr = srq_attr->max_wr; - cmd->srq_limit = srq_attr->srq_limit; - - if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - return 0; -} - -int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - struct ibv_query_srq *cmd, size_t cmd_size) -{ - struct ibv_query_srq_resp resp; - - IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp); - cmd->srq_handle = srq->handle; - cmd->reserved = 0; - - if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - srq_attr->max_wr = resp.max_wr; - srq_attr->max_sge = resp.max_sge; - srq_attr->srq_limit = resp.srq_limit; - - return 0; -} - -int ibv_cmd_destroy_srq(struct ibv_srq *srq) -{ - struct ibv_destroy_srq cmd; - struct ibv_destroy_srq_resp resp; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp); - cmd.srq_handle = srq->handle; - cmd.reserved = 0; - - if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - pthread_mutex_lock(&srq->mutex); - while (srq->events_completed != resp.events_reported) - pthread_cond_wait(&srq->cond, &srq->mutex); - pthread_mutex_unlock(&srq->mutex); - - return 0; -} - -static int create_qp_ex_common(struct verbs_qp *qp, - struct ibv_qp_init_attr_ex *qp_attr, - struct verbs_xrcd *vxrcd, - struct ibv_create_qp_common *cmd) -{ - cmd->user_handle = (uintptr_t)qp; - - if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD) { - vxrcd = container_of(qp_attr->xrcd, struct verbs_xrcd, xrcd); - cmd->pd_handle = vxrcd->handle; - } else { - if (!(qp_attr->comp_mask & IBV_QP_INIT_ATTR_PD)) - return EINVAL; - - cmd->pd_handle = qp_attr->pd->handle; - if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { - if (cmd->max_recv_wr || cmd->max_recv_sge || - cmd->recv_cq_handle || qp_attr->srq) - return EINVAL; - - /* send_cq is optinal */ - if (qp_attr->cap.max_send_wr) - cmd->send_cq_handle = qp_attr->send_cq->handle; - } else { - cmd->send_cq_handle = qp_attr->send_cq->handle; - - if (qp_attr->qp_type != IBV_QPT_XRC_SEND) { - cmd->recv_cq_handle = qp_attr->recv_cq->handle; - cmd->srq_handle = qp_attr->srq ? qp_attr->srq->handle : - 0; - } - } - } - - cmd->max_send_wr = qp_attr->cap.max_send_wr; - cmd->max_recv_wr = qp_attr->cap.max_recv_wr; - cmd->max_send_sge = qp_attr->cap.max_send_sge; - cmd->max_recv_sge = qp_attr->cap.max_recv_sge; - cmd->max_inline_data = qp_attr->cap.max_inline_data; - cmd->sq_sig_all = qp_attr->sq_sig_all; - cmd->qp_type = qp_attr->qp_type; - cmd->is_srq = !!qp_attr->srq; - cmd->reserved = 0; - - return 0; -} - -static void create_qp_handle_resp_common(struct ibv_context *context, - struct verbs_qp *qp, - struct ibv_qp_init_attr_ex *qp_attr, - struct ibv_create_qp_resp *resp, - struct verbs_xrcd *vxrcd, - int vqp_sz) -{ - if (abi_ver > 3) { - qp_attr->cap.max_recv_sge = resp->max_recv_sge; - qp_attr->cap.max_send_sge = resp->max_send_sge; - qp_attr->cap.max_recv_wr = resp->max_recv_wr; - qp_attr->cap.max_send_wr = resp->max_send_wr; - qp_attr->cap.max_inline_data = resp->max_inline_data; - } - - qp->qp.handle = resp->qp_handle; - qp->qp.qp_num = resp->qpn; - qp->qp.context = context; - qp->qp.qp_context = qp_attr->qp_context; - qp->qp.pd = qp_attr->pd; - qp->qp.send_cq = qp_attr->send_cq; - qp->qp.recv_cq = qp_attr->recv_cq; - qp->qp.srq = qp_attr->srq; - qp->qp.qp_type = qp_attr->qp_type; - qp->qp.state = IBV_QPS_RESET; - qp->qp.events_completed = 0; - pthread_mutex_init(&qp->qp.mutex, NULL); - pthread_cond_init(&qp->qp.cond, NULL); - - qp->comp_mask = 0; - if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) && - (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) { - qp->comp_mask |= VERBS_QP_XRCD; - qp->xrcd = vxrcd; - } -} - -enum { - CREATE_QP_EX2_SUP_CREATE_FLAGS = IBV_QP_CREATE_BLOCK_SELF_MCAST_LB | - IBV_QP_CREATE_SCATTER_FCS, -}; - -int ibv_cmd_create_qp_ex2(struct ibv_context *context, - struct verbs_qp *qp, int vqp_sz, - struct ibv_qp_init_attr_ex *qp_attr, - struct ibv_create_qp_ex *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_qp_resp_ex *resp, - size_t resp_core_size, - size_t resp_size) -{ - struct verbs_xrcd *vxrcd = NULL; - int err; - - if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED) - return EINVAL; - - if (resp_core_size < - offsetof(struct ibv_create_qp_resp_ex, response_length) + - sizeof(resp->response_length)) - return EINVAL; - - memset(cmd, 0, cmd_core_size); - - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_QP_EX, resp, - resp_core_size, resp_size); - - err = create_qp_ex_common(qp, qp_attr, vxrcd, &cmd->base); - if (err) - return err; - - if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) { - if (qp_attr->create_flags & ~CREATE_QP_EX2_SUP_CREATE_FLAGS) - return EINVAL; - if (cmd_core_size < offsetof(struct ibv_create_qp_ex, create_flags) + - sizeof(qp_attr->create_flags)) - return EINVAL; - cmd->create_flags = qp_attr->create_flags; - } - - if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { - if (cmd_core_size < offsetof(struct ibv_create_qp_ex, ind_tbl_handle) + - sizeof(cmd->ind_tbl_handle)) - return EINVAL; - cmd->ind_tbl_handle = qp_attr->rwq_ind_tbl->ind_tbl_handle; - cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE; - } - - err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; - - (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - create_qp_handle_resp_common(context, qp, qp_attr, &resp->base, vxrcd, - vqp_sz); - - return 0; -} - -int ibv_cmd_create_qp_ex(struct ibv_context *context, - struct verbs_qp *qp, int vqp_sz, - struct ibv_qp_init_attr_ex *attr_ex, - struct ibv_create_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size) -{ - struct verbs_xrcd *vxrcd = NULL; - int err; - - IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); - - if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD)) - return ENOSYS; - - err = create_qp_ex_common(qp, attr_ex, vxrcd, - (struct ibv_create_qp_common *)&cmd->user_handle); - if (err) - return err; - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - if (abi_ver == 4) { - struct ibv_create_qp_resp_v4 *resp_v4 = - (struct ibv_create_qp_resp_v4 *)resp; - - memmove((void *)resp + sizeof *resp, - (void *)resp_v4 + sizeof *resp_v4, - resp_size - sizeof *resp); - } else if (abi_ver <= 3) { - struct ibv_create_qp_resp_v3 *resp_v3 = - (struct ibv_create_qp_resp_v3 *)resp; - - memmove((void *)resp + sizeof *resp, - (void *)resp_v3 + sizeof *resp_v3, - resp_size - sizeof *resp); - } - - create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz); - - return 0; -} - -int ibv_cmd_create_qp(struct ibv_pd *pd, - struct ibv_qp *qp, struct ibv_qp_init_attr *attr, - struct ibv_create_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size) -{ - IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); - - cmd->user_handle = (uintptr_t) qp; - cmd->pd_handle = pd->handle; - cmd->send_cq_handle = attr->send_cq->handle; - cmd->recv_cq_handle = attr->recv_cq->handle; - cmd->srq_handle = attr->srq ? attr->srq->handle : 0; - cmd->max_send_wr = attr->cap.max_send_wr; - cmd->max_recv_wr = attr->cap.max_recv_wr; - cmd->max_send_sge = attr->cap.max_send_sge; - cmd->max_recv_sge = attr->cap.max_recv_sge; - cmd->max_inline_data = attr->cap.max_inline_data; - cmd->sq_sig_all = attr->sq_sig_all; - cmd->qp_type = attr->qp_type; - cmd->is_srq = !!attr->srq; - cmd->reserved = 0; - - if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - qp->handle = resp->qp_handle; - qp->qp_num = resp->qpn; - qp->context = pd->context; - - if (abi_ver > 3) { - attr->cap.max_recv_sge = resp->max_recv_sge; - attr->cap.max_send_sge = resp->max_send_sge; - attr->cap.max_recv_wr = resp->max_recv_wr; - attr->cap.max_send_wr = resp->max_send_wr; - attr->cap.max_inline_data = resp->max_inline_data; - } - - if (abi_ver == 4) { - struct ibv_create_qp_resp_v4 *resp_v4 = - (struct ibv_create_qp_resp_v4 *) resp; - - memmove((void *) resp + sizeof *resp, - (void *) resp_v4 + sizeof *resp_v4, - resp_size - sizeof *resp); - } else if (abi_ver <= 3) { - struct ibv_create_qp_resp_v3 *resp_v3 = - (struct ibv_create_qp_resp_v3 *) resp; - - memmove((void *) resp + sizeof *resp, - (void *) resp_v3 + sizeof *resp_v3, - resp_size - sizeof *resp); - } - - return 0; -} - -int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, - int vqp_sz, - struct ibv_qp_open_attr *attr, - struct ibv_open_qp *cmd, size_t cmd_size, - struct ibv_create_qp_resp *resp, size_t resp_size) -{ - struct verbs_xrcd *xrcd; - IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size); - - if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED) - return ENOSYS; - - if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) || - !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) || - !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE)) - return EINVAL; - - xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd); - cmd->user_handle = (uintptr_t) qp; - cmd->pd_handle = xrcd->handle; - cmd->qpn = attr->qp_num; - cmd->qp_type = attr->qp_type; - - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - qp->qp.handle = resp->qp_handle; - qp->qp.context = context; - qp->qp.qp_context = attr->qp_context; - qp->qp.pd = NULL; - qp->qp.send_cq = NULL; - qp->qp.recv_cq = NULL; - qp->qp.srq = NULL; - qp->qp.qp_num = attr->qp_num; - qp->qp.qp_type = attr->qp_type; - qp->qp.state = IBV_QPS_UNKNOWN; - qp->qp.events_completed = 0; - pthread_mutex_init(&qp->qp.mutex, NULL); - pthread_cond_init(&qp->qp.cond, NULL); - qp->comp_mask = 0; - if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) { - qp->comp_mask = VERBS_QP_XRCD; - qp->xrcd = xrcd; - } - - return 0; -} - -int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr, - struct ibv_query_qp *cmd, size_t cmd_size) -{ - struct ibv_query_qp_resp resp; - - IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp); - cmd->qp_handle = qp->handle; - cmd->attr_mask = attr_mask; - - if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - attr->qkey = resp.qkey; - attr->rq_psn = resp.rq_psn; - attr->sq_psn = resp.sq_psn; - attr->dest_qp_num = resp.dest_qp_num; - attr->qp_access_flags = resp.qp_access_flags; - attr->pkey_index = resp.pkey_index; - attr->alt_pkey_index = resp.alt_pkey_index; - attr->qp_state = resp.qp_state; - attr->cur_qp_state = resp.cur_qp_state; - attr->path_mtu = resp.path_mtu; - attr->path_mig_state = resp.path_mig_state; - attr->sq_draining = resp.sq_draining; - attr->max_rd_atomic = resp.max_rd_atomic; - attr->max_dest_rd_atomic = resp.max_dest_rd_atomic; - attr->min_rnr_timer = resp.min_rnr_timer; - attr->port_num = resp.port_num; - attr->timeout = resp.timeout; - attr->retry_cnt = resp.retry_cnt; - attr->rnr_retry = resp.rnr_retry; - attr->alt_port_num = resp.alt_port_num; - attr->alt_timeout = resp.alt_timeout; - attr->cap.max_send_wr = resp.max_send_wr; - attr->cap.max_recv_wr = resp.max_recv_wr; - attr->cap.max_send_sge = resp.max_send_sge; - attr->cap.max_recv_sge = resp.max_recv_sge; - attr->cap.max_inline_data = resp.max_inline_data; - - memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16); - attr->ah_attr.grh.flow_label = resp.dest.flow_label; - attr->ah_attr.dlid = resp.dest.dlid; - attr->ah_attr.grh.sgid_index = resp.dest.sgid_index; - attr->ah_attr.grh.hop_limit = resp.dest.hop_limit; - attr->ah_attr.grh.traffic_class = resp.dest.traffic_class; - attr->ah_attr.sl = resp.dest.sl; - attr->ah_attr.src_path_bits = resp.dest.src_path_bits; - attr->ah_attr.static_rate = resp.dest.static_rate; - attr->ah_attr.is_global = resp.dest.is_global; - attr->ah_attr.port_num = resp.dest.port_num; - - memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16); - attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label; - attr->alt_ah_attr.dlid = resp.alt_dest.dlid; - attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index; - attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit; - attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class; - attr->alt_ah_attr.sl = resp.alt_dest.sl; - attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits; - attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate; - attr->alt_ah_attr.is_global = resp.alt_dest.is_global; - attr->alt_ah_attr.port_num = resp.alt_dest.port_num; - - init_attr->qp_context = qp->qp_context; - init_attr->send_cq = qp->send_cq; - init_attr->recv_cq = qp->recv_cq; - init_attr->srq = qp->srq; - init_attr->qp_type = qp->qp_type; - init_attr->cap.max_send_wr = resp.max_send_wr; - init_attr->cap.max_recv_wr = resp.max_recv_wr; - init_attr->cap.max_send_sge = resp.max_send_sge; - init_attr->cap.max_recv_sge = resp.max_recv_sge; - init_attr->cap.max_inline_data = resp.max_inline_data; - init_attr->sq_sig_all = resp.sq_sig_all; - - return 0; -} - -int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_modify_qp *cmd, size_t cmd_size) -{ - IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP); - - cmd->qp_handle = qp->handle; - cmd->attr_mask = attr_mask; - cmd->qkey = attr->qkey; - cmd->rq_psn = attr->rq_psn; - cmd->sq_psn = attr->sq_psn; - cmd->dest_qp_num = attr->dest_qp_num; - cmd->qp_access_flags = attr->qp_access_flags; - cmd->pkey_index = attr->pkey_index; - cmd->alt_pkey_index = attr->alt_pkey_index; - cmd->qp_state = attr->qp_state; - cmd->cur_qp_state = attr->cur_qp_state; - cmd->path_mtu = attr->path_mtu; - cmd->path_mig_state = attr->path_mig_state; - cmd->en_sqd_async_notify = attr->en_sqd_async_notify; - cmd->max_rd_atomic = attr->max_rd_atomic; - cmd->max_dest_rd_atomic = attr->max_dest_rd_atomic; - cmd->min_rnr_timer = attr->min_rnr_timer; - cmd->port_num = attr->port_num; - cmd->timeout = attr->timeout; - cmd->retry_cnt = attr->retry_cnt; - cmd->rnr_retry = attr->rnr_retry; - cmd->alt_port_num = attr->alt_port_num; - cmd->alt_timeout = attr->alt_timeout; - - memcpy(cmd->dest.dgid, attr->ah_attr.grh.dgid.raw, 16); - cmd->dest.flow_label = attr->ah_attr.grh.flow_label; - cmd->dest.dlid = attr->ah_attr.dlid; - cmd->dest.reserved = 0; - cmd->dest.sgid_index = attr->ah_attr.grh.sgid_index; - cmd->dest.hop_limit = attr->ah_attr.grh.hop_limit; - cmd->dest.traffic_class = attr->ah_attr.grh.traffic_class; - cmd->dest.sl = attr->ah_attr.sl; - cmd->dest.src_path_bits = attr->ah_attr.src_path_bits; - cmd->dest.static_rate = attr->ah_attr.static_rate; - cmd->dest.is_global = attr->ah_attr.is_global; - cmd->dest.port_num = attr->ah_attr.port_num; - - memcpy(cmd->alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); - cmd->alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; - cmd->alt_dest.dlid = attr->alt_ah_attr.dlid; - cmd->alt_dest.reserved = 0; - cmd->alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; - cmd->alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; - cmd->alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; - cmd->alt_dest.sl = attr->alt_ah_attr.sl; - cmd->alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; - cmd->alt_dest.static_rate = attr->alt_ah_attr.static_rate; - cmd->alt_dest.is_global = attr->alt_ah_attr.is_global; - cmd->alt_dest.port_num = attr->alt_ah_attr.port_num; - - cmd->reserved[0] = cmd->reserved[1] = 0; - - if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - return 0; -} - -int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, - struct ibv_send_wr **bad_wr) -{ - struct ibv_post_send *cmd; - struct ibv_post_send_resp resp; - struct ibv_send_wr *i; - struct ibv_kern_send_wr *n, *tmp; - struct ibv_sge *s; - unsigned wr_count = 0; - unsigned sge_count = 0; - int cmd_size; - int ret = 0; - - for (i = wr; i; i = i->next) { - wr_count++; - sge_count += i->num_sge; - } - - cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; - cmd = alloca(cmd_size); - - IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SEND, &resp, sizeof resp); - cmd->qp_handle = ibqp->handle; - cmd->wr_count = wr_count; - cmd->sge_count = sge_count; - cmd->wqe_size = sizeof *n; - - n = (struct ibv_kern_send_wr *) ((void *) cmd + sizeof *cmd); - s = (struct ibv_sge *) (n + wr_count); - - tmp = n; - for (i = wr; i; i = i->next) { - tmp->wr_id = i->wr_id; - tmp->num_sge = i->num_sge; - tmp->opcode = i->opcode; - tmp->send_flags = i->send_flags; - tmp->imm_data = i->imm_data; - if (ibqp->qp_type == IBV_QPT_UD) { - tmp->wr.ud.ah = i->wr.ud.ah->handle; - tmp->wr.ud.remote_qpn = i->wr.ud.remote_qpn; - tmp->wr.ud.remote_qkey = i->wr.ud.remote_qkey; - } else { - switch (i->opcode) { - case IBV_WR_RDMA_WRITE: - case IBV_WR_RDMA_WRITE_WITH_IMM: - case IBV_WR_RDMA_READ: - tmp->wr.rdma.remote_addr = - i->wr.rdma.remote_addr; - tmp->wr.rdma.rkey = i->wr.rdma.rkey; - break; - case IBV_WR_ATOMIC_CMP_AND_SWP: - case IBV_WR_ATOMIC_FETCH_AND_ADD: - tmp->wr.atomic.remote_addr = - i->wr.atomic.remote_addr; - tmp->wr.atomic.compare_add = - i->wr.atomic.compare_add; - tmp->wr.atomic.swap = i->wr.atomic.swap; - tmp->wr.atomic.rkey = i->wr.atomic.rkey; - break; - default: - break; - } - } - - if (tmp->num_sge) { - memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); - s += tmp->num_sge; - } - - tmp++; - } - - resp.bad_wr = 0; - if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) - ret = errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - wr_count = resp.bad_wr; - if (wr_count) { - i = wr; - while (--wr_count) - i = i->next; - *bad_wr = i; - } else if (ret) - *bad_wr = wr; - - return ret; -} - -int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr) -{ - struct ibv_post_recv *cmd; - struct ibv_post_recv_resp resp; - struct ibv_recv_wr *i; - struct ibv_kern_recv_wr *n, *tmp; - struct ibv_sge *s; - unsigned wr_count = 0; - unsigned sge_count = 0; - int cmd_size; - int ret = 0; - - for (i = wr; i; i = i->next) { - wr_count++; - sge_count += i->num_sge; - } - - cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; - cmd = alloca(cmd_size); - - IBV_INIT_CMD_RESP(cmd, cmd_size, POST_RECV, &resp, sizeof resp); - cmd->qp_handle = ibqp->handle; - cmd->wr_count = wr_count; - cmd->sge_count = sge_count; - cmd->wqe_size = sizeof *n; - - n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); - s = (struct ibv_sge *) (n + wr_count); - - tmp = n; - for (i = wr; i; i = i->next) { - tmp->wr_id = i->wr_id; - tmp->num_sge = i->num_sge; - - if (tmp->num_sge) { - memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); - s += tmp->num_sge; - } - - tmp++; - } - - resp.bad_wr = 0; - if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) - ret = errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - wr_count = resp.bad_wr; - if (wr_count) { - i = wr; - while (--wr_count) - i = i->next; - *bad_wr = i; - } else if (ret) - *bad_wr = wr; - - return ret; -} - -int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, - struct ibv_recv_wr **bad_wr) -{ - struct ibv_post_srq_recv *cmd; - struct ibv_post_srq_recv_resp resp; - struct ibv_recv_wr *i; - struct ibv_kern_recv_wr *n, *tmp; - struct ibv_sge *s; - unsigned wr_count = 0; - unsigned sge_count = 0; - int cmd_size; - int ret = 0; - - for (i = wr; i; i = i->next) { - wr_count++; - sge_count += i->num_sge; - } - - cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; - cmd = alloca(cmd_size); - - IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SRQ_RECV, &resp, sizeof resp); - cmd->srq_handle = srq->handle; - cmd->wr_count = wr_count; - cmd->sge_count = sge_count; - cmd->wqe_size = sizeof *n; - - n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); - s = (struct ibv_sge *) (n + wr_count); - - tmp = n; - for (i = wr; i; i = i->next) { - tmp->wr_id = i->wr_id; - tmp->num_sge = i->num_sge; - - if (tmp->num_sge) { - memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); - s += tmp->num_sge; - } - - tmp++; - } - - resp.bad_wr = 0; - if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) - ret = errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - wr_count = resp.bad_wr; - if (wr_count) { - i = wr; - while (--wr_count) - i = i->next; - *bad_wr = i; - } else if (ret) - *bad_wr = wr; - - return ret; -} - -int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, - struct ibv_ah_attr *attr) -{ - struct ibv_create_ah cmd; - struct ibv_create_ah_resp resp; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_AH, &resp, sizeof resp); - cmd.user_handle = (uintptr_t) ah; - cmd.pd_handle = pd->handle; - cmd.attr.dlid = attr->dlid; - cmd.attr.sl = attr->sl; - cmd.attr.src_path_bits = attr->src_path_bits; - cmd.attr.static_rate = attr->static_rate; - cmd.attr.is_global = attr->is_global; - cmd.attr.port_num = attr->port_num; - cmd.attr.grh.flow_label = attr->grh.flow_label; - cmd.attr.grh.sgid_index = attr->grh.sgid_index; - cmd.attr.grh.hop_limit = attr->grh.hop_limit; - cmd.attr.grh.traffic_class = attr->grh.traffic_class; - memcpy(cmd.attr.grh.dgid, attr->grh.dgid.raw, 16); - - if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - ah->handle = resp.handle; - ah->context = pd->context; - - return 0; -} - -int ibv_cmd_destroy_ah(struct ibv_ah *ah) -{ - struct ibv_destroy_ah cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_AH); - cmd.ah_handle = ah->handle; - - if (write(ah->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_destroy_qp(struct ibv_qp *qp) -{ - struct ibv_destroy_qp cmd; - struct ibv_destroy_qp_resp resp; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp); - cmd.qp_handle = qp->handle; - cmd.reserved = 0; - - if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - pthread_mutex_lock(&qp->mutex); - while (qp->events_completed != resp.events_reported) - pthread_cond_wait(&qp->cond, &qp->mutex); - pthread_mutex_unlock(&qp->mutex); - - return 0; -} - -int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) -{ - struct ibv_attach_mcast cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, ATTACH_MCAST); - memcpy(cmd.gid, gid->raw, sizeof cmd.gid); - cmd.qp_handle = qp->handle; - cmd.mlid = lid; - cmd.reserved = 0; - - if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) -{ - struct ibv_detach_mcast cmd; - - IBV_INIT_CMD(&cmd, sizeof cmd, DETACH_MCAST); - memcpy(cmd.gid, gid->raw, sizeof cmd.gid); - cmd.qp_handle = qp->handle; - cmd.mlid = lid; - cmd.reserved = 0; - - if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) - return errno; - - return 0; -} - -static int ib_spec_to_kern_spec(struct ibv_flow_spec *ib_spec, - struct ibv_kern_spec *kern_spec) -{ - kern_spec->hdr.type = ib_spec->hdr.type; - - switch (ib_spec->hdr.type) { - case IBV_FLOW_SPEC_ETH: - kern_spec->eth.size = sizeof(struct ibv_kern_spec_eth); - memcpy(&kern_spec->eth.val, &ib_spec->eth.val, - sizeof(struct ibv_flow_eth_filter)); - memcpy(&kern_spec->eth.mask, &ib_spec->eth.mask, - sizeof(struct ibv_flow_eth_filter)); - break; - case IBV_FLOW_SPEC_IPV4: - kern_spec->ipv4.size = sizeof(struct ibv_kern_spec_ipv4); - memcpy(&kern_spec->ipv4.val, &ib_spec->ipv4.val, - sizeof(struct ibv_flow_ipv4_filter)); - memcpy(&kern_spec->ipv4.mask, &ib_spec->ipv4.mask, - sizeof(struct ibv_flow_ipv4_filter)); - break; - case IBV_FLOW_SPEC_TCP: - case IBV_FLOW_SPEC_UDP: - kern_spec->tcp_udp.size = sizeof(struct ibv_kern_spec_tcp_udp); - memcpy(&kern_spec->tcp_udp.val, &ib_spec->tcp_udp.val, - sizeof(struct ibv_flow_ipv4_filter)); - memcpy(&kern_spec->tcp_udp.mask, &ib_spec->tcp_udp.mask, - sizeof(struct ibv_flow_tcp_udp_filter)); - break; - default: - return -EINVAL; - } - return 0; -} - -struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp, - struct ibv_flow_attr *flow_attr) -{ - struct ibv_create_flow *cmd; - struct ibv_create_flow_resp resp; - struct ibv_flow *flow_id; - size_t cmd_size; - size_t written_size; - int i, err; - void *kern_spec; - void *ib_spec; - - cmd_size = sizeof(*cmd) + (flow_attr->num_of_specs * - sizeof(struct ibv_kern_spec)); - cmd = alloca(cmd_size); - flow_id = malloc(sizeof(*flow_id)); - if (!flow_id) - return NULL; - memset(cmd, 0, cmd_size); - - cmd->qp_handle = qp->handle; - - cmd->flow_attr.type = flow_attr->type; - cmd->flow_attr.priority = flow_attr->priority; - cmd->flow_attr.num_of_specs = flow_attr->num_of_specs; - cmd->flow_attr.port = flow_attr->port; - cmd->flow_attr.flags = flow_attr->flags; - - kern_spec = cmd + 1; - ib_spec = flow_attr + 1; - for (i = 0; i < flow_attr->num_of_specs; i++) { - err = ib_spec_to_kern_spec(ib_spec, kern_spec); - if (err) - goto err; - cmd->flow_attr.size += - ((struct ibv_kern_spec *)kern_spec)->hdr.size; - kern_spec += ((struct ibv_kern_spec *)kern_spec)->hdr.size; - ib_spec += ((struct ibv_flow_spec *)ib_spec)->hdr.size; - } - - written_size = sizeof(*cmd) + cmd->flow_attr.size; - IBV_INIT_CMD_RESP_EX_VCMD(cmd, written_size, written_size, CREATE_FLOW, - &resp, sizeof(resp)); - if (write(qp->context->cmd_fd, cmd, written_size) != written_size) - goto err; - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp)); - - flow_id->context = qp->context; - flow_id->handle = resp.flow_handle; - return flow_id; -err: - free(flow_id); - return NULL; -} - -int ibv_cmd_destroy_flow(struct ibv_flow *flow_id) -{ - struct ibv_destroy_flow cmd; - int ret = 0; - - memset(&cmd, 0, sizeof(cmd)); - IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_FLOW); - cmd.flow_handle = flow_id->handle; - - if (write(flow_id->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) - ret = errno; - free(flow_id); - return ret; -} - -int ibv_cmd_create_wq(struct ibv_context *context, - struct ibv_wq_init_attr *wq_init_attr, - struct ibv_wq *wq, - struct ibv_create_wq *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_wq_resp *resp, - size_t resp_core_size, - size_t resp_size) -{ - int err; - - if (wq_init_attr->comp_mask >= IBV_WQ_INIT_ATTR_RESERVED) - return EINVAL; - - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, - CREATE_WQ, resp, - resp_core_size, resp_size); - - cmd->user_handle = (uintptr_t)wq; - cmd->pd_handle = wq_init_attr->pd->handle; - cmd->cq_handle = wq_init_attr->cq->handle; - cmd->wq_type = wq_init_attr->wq_type; - cmd->max_sge = wq_init_attr->max_sge; - cmd->max_wr = wq_init_attr->max_wr; - cmd->comp_mask = 0; - - err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - if (resp->response_length < resp_core_size) - return EINVAL; - - wq->handle = resp->wq_handle; - wq_init_attr->max_wr = resp->max_wr; - wq_init_attr->max_sge = resp->max_sge; - wq->wq_num = resp->wqn; - wq->context = context; - wq->cq = wq_init_attr->cq; - wq->pd = wq_init_attr->pd; - wq->wq_type = wq_init_attr->wq_type; - - return 0; -} - -int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr, - struct ibv_modify_wq *cmd, size_t cmd_core_size, - size_t cmd_size) -{ - if (attr->attr_mask >= IBV_WQ_ATTR_RESERVED) - return EINVAL; - - memset(cmd, 0, cmd_core_size); - IBV_INIT_CMD_EX(cmd, cmd_size, MODIFY_WQ); - - cmd->curr_wq_state = attr->curr_wq_state; - cmd->wq_state = attr->wq_state; - cmd->wq_handle = wq->handle; - cmd->attr_mask = attr->attr_mask; - - if (write(wq->context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; - - if (attr->attr_mask & IBV_WQ_ATTR_STATE) - wq->state = attr->wq_state; - - return 0; -} - -int ibv_cmd_destroy_wq(struct ibv_wq *wq) -{ - struct ibv_destroy_wq cmd; - struct ibv_destroy_wq_resp resp; - int ret = 0; - - memset(&cmd, 0, sizeof(cmd)); - memset(&resp, 0, sizeof(resp)); - - IBV_INIT_CMD_RESP_EX(&cmd, sizeof(cmd), DESTROY_WQ, &resp, sizeof(resp)); - cmd.wq_handle = wq->handle; - - if (write(wq->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) - return errno; - - if (resp.response_length < sizeof(resp)) - return EINVAL; - - pthread_mutex_lock(&wq->mutex); - while (wq->events_completed != resp.events_reported) - pthread_cond_wait(&wq->cond, &wq->mutex); - pthread_mutex_unlock(&wq->mutex); - - return ret; -} - -int ibv_cmd_create_rwq_ind_table(struct ibv_context *context, - struct ibv_rwq_ind_table_init_attr *init_attr, - struct ibv_rwq_ind_table *rwq_ind_table, - struct ibv_create_rwq_ind_table *cmd, - size_t cmd_core_size, - size_t cmd_size, - struct ibv_create_rwq_ind_table_resp *resp, - size_t resp_core_size, - size_t resp_size) -{ - int err, i; - uint32_t required_tbl_size, alloc_tbl_size; - uint32_t *tbl_start; - int num_tbl_entries; - - if (init_attr->comp_mask >= IBV_CREATE_IND_TABLE_RESERVED) - return EINVAL; - - alloc_tbl_size = cmd_core_size - sizeof(*cmd); - num_tbl_entries = 1 << init_attr->log_ind_tbl_size; - - /* Data must be u64 aligned */ - required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ? - sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t)); - - if (alloc_tbl_size < required_tbl_size) - return EINVAL; - - tbl_start = (uint32_t *)((uint8_t *)cmd + sizeof(*cmd)); - for (i = 0; i < num_tbl_entries; i++) - tbl_start[i] = init_attr->ind_tbl[i]->handle; - - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, - CREATE_RWQ_IND_TBL, resp, - resp_core_size, resp_size); - cmd->log_ind_tbl_size = init_attr->log_ind_tbl_size; - cmd->comp_mask = 0; - - err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; - - (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); - - if (resp->response_length < resp_core_size) - return EINVAL; - - rwq_ind_table->ind_tbl_handle = resp->ind_tbl_handle; - rwq_ind_table->ind_tbl_num = resp->ind_tbl_num; - rwq_ind_table->context = context; - return 0; -} - -int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) -{ - struct ibv_destroy_rwq_ind_table cmd; - int ret = 0; - - memset(&cmd, 0, sizeof(cmd)); - IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_RWQ_IND_TBL); - cmd.ind_tbl_handle = rwq_ind_table->ind_tbl_handle; - - if (write(rwq_ind_table->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) - ret = errno; - - return ret; -} diff --git a/src/compat-1_0.c b/src/compat-1_0.c deleted file mode 100644 index 6b1961d..0000000 --- a/src/compat-1_0.c +++ /dev/null @@ -1,939 +0,0 @@ -/* - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <string.h> -#include <stddef.h> -#include <stdlib.h> -#include <unistd.h> -#include <alloca.h> - -#include "ibverbs.h" - -struct ibv_pd_1_0 { - struct ibv_context_1_0 *context; - uint32_t handle; - - struct ibv_pd *real_pd; -}; - -struct ibv_mr_1_0 { - struct ibv_context_1_0 *context; - struct ibv_pd_1_0 *pd; - uint32_t handle; - uint32_t lkey; - uint32_t rkey; - - struct ibv_mr *real_mr; -}; - -struct ibv_srq_1_0 { - struct ibv_context_1_0 *context; - void *srq_context; - struct ibv_pd_1_0 *pd; - uint32_t handle; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t events_completed; - - struct ibv_srq *real_srq; -}; - -struct ibv_qp_init_attr_1_0 { - void *qp_context; - struct ibv_cq_1_0 *send_cq; - struct ibv_cq_1_0 *recv_cq; - struct ibv_srq_1_0 *srq; - struct ibv_qp_cap cap; - enum ibv_qp_type qp_type; - int sq_sig_all; -}; - -struct ibv_send_wr_1_0 { - struct ibv_send_wr_1_0 *next; - uint64_t wr_id; - struct ibv_sge *sg_list; - int num_sge; - enum ibv_wr_opcode opcode; - int send_flags; - uint32_t imm_data; /* in network byte order */ - union { - struct { - uint64_t remote_addr; - uint32_t rkey; - } rdma; - struct { - uint64_t remote_addr; - uint64_t compare_add; - uint64_t swap; - uint32_t rkey; - } atomic; - struct { - struct ibv_ah_1_0 *ah; - uint32_t remote_qpn; - uint32_t remote_qkey; - } ud; - } wr; -}; - -struct ibv_recv_wr_1_0 { - struct ibv_recv_wr_1_0 *next; - uint64_t wr_id; - struct ibv_sge *sg_list; - int num_sge; -}; - -struct ibv_qp_1_0 { - struct ibv_context_1_0 *context; - void *qp_context; - struct ibv_pd_1_0 *pd; - struct ibv_cq_1_0 *send_cq; - struct ibv_cq_1_0 *recv_cq; - struct ibv_srq_1_0 *srq; - uint32_t handle; - uint32_t qp_num; - enum ibv_qp_state state; - enum ibv_qp_type qp_type; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t events_completed; - - struct ibv_qp *real_qp; -}; - -struct ibv_cq_1_0 { - struct ibv_context_1_0 *context; - void *cq_context; - uint32_t handle; - int cqe; - - pthread_mutex_t mutex; - pthread_cond_t cond; - uint32_t comp_events_completed; - uint32_t async_events_completed; - - struct ibv_cq *real_cq; -}; - -struct ibv_ah_1_0 { - struct ibv_context_1_0 *context; - struct ibv_pd_1_0 *pd; - uint32_t handle; - - struct ibv_ah *real_ah; -}; - -struct ibv_device_1_0 { - void *obsolete_sysfs_dev; - void *obsolete_sysfs_ibdev; - struct ibv_device *real_device; /* was obsolete driver member */ - struct ibv_device_ops ops; -}; - -struct ibv_context_ops_1_0 { - int (*query_device)(struct ibv_context *context, - struct ibv_device_attr *device_attr); - int (*query_port)(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr); - struct ibv_pd * (*alloc_pd)(struct ibv_context *context); - int (*dealloc_pd)(struct ibv_pd *pd); - struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - int access); - int (*dereg_mr)(struct ibv_mr *mr); - struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, - struct ibv_comp_channel *channel, - int comp_vector); - int (*poll_cq)(struct ibv_cq_1_0 *cq, int num_entries, - struct ibv_wc *wc); - int (*req_notify_cq)(struct ibv_cq_1_0 *cq, - int solicited_only); - void (*cq_event)(struct ibv_cq *cq); - int (*resize_cq)(struct ibv_cq *cq, int cqe); - int (*destroy_cq)(struct ibv_cq *cq); - struct ibv_srq * (*create_srq)(struct ibv_pd *pd, - struct ibv_srq_init_attr *srq_init_attr); - int (*modify_srq)(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask); - int (*query_srq)(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr); - int (*destroy_srq)(struct ibv_srq *srq); - int (*post_srq_recv)(struct ibv_srq_1_0 *srq, - struct ibv_recv_wr_1_0 *recv_wr, - struct ibv_recv_wr_1_0 **bad_recv_wr); - struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); - int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr); - int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask); - int (*destroy_qp)(struct ibv_qp *qp); - int (*post_send)(struct ibv_qp_1_0 *qp, - struct ibv_send_wr_1_0 *wr, - struct ibv_send_wr_1_0 **bad_wr); - int (*post_recv)(struct ibv_qp_1_0 *qp, - struct ibv_recv_wr_1_0 *wr, - struct ibv_recv_wr_1_0 **bad_wr); - struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); - int (*destroy_ah)(struct ibv_ah *ah); - int (*attach_mcast)(struct ibv_qp *qp, union ibv_gid *gid, - uint16_t lid); - int (*detach_mcast)(struct ibv_qp *qp, union ibv_gid *gid, - uint16_t lid); -}; - -struct ibv_context_1_0 { - struct ibv_device_1_0 *device; - struct ibv_context_ops_1_0 ops; - int cmd_fd; - int async_fd; - int num_comp_vectors; - - struct ibv_context *real_context; /* was abi_compat member */ -}; - -struct ibv_device_1_0 **__ibv_get_device_list_1_0(int *num) -{ - struct ibv_device **real_list; - struct ibv_device_1_0 **l; - int i, n; - - real_list = ibv_get_device_list(&n); - if (!real_list) - return NULL; - - l = calloc(n + 2, sizeof (struct ibv_device_1_0 *)); - if (!l) - goto free_device_list; - - l[0] = (void *) real_list; - - for (i = 0; i < n; ++i) { - l[i + 1] = calloc(1, sizeof (struct ibv_device_1_0)); - if (!l[i + 1]) - goto fail; - l[i + 1]->real_device = real_list[i]; - } - - if (num) - *num = n; - - return l + 1; - -fail: - for (i = 1; i <= n; ++i) - if (l[i]) - free(l[i]); - free(l); - -free_device_list: - ibv_free_device_list(real_list); - return NULL; -} -symver(__ibv_get_device_list_1_0, ibv_get_device_list, IBVERBS_1.0); - -void __ibv_free_device_list_1_0(struct ibv_device_1_0 **list) -{ - struct ibv_device_1_0 **l = list; - - while (*l) { - free(*l); - ++l; - } - - ibv_free_device_list((void *) list[-1]); - free(list - 1); -} -symver(__ibv_free_device_list_1_0, ibv_free_device_list, IBVERBS_1.0); - -const char *__ibv_get_device_name_1_0(struct ibv_device_1_0 *device) -{ - return ibv_get_device_name(device->real_device); -} -symver(__ibv_get_device_name_1_0, ibv_get_device_name, IBVERBS_1.0); - -uint64_t __ibv_get_device_guid_1_0(struct ibv_device_1_0 *device) -{ - return ibv_get_device_guid(device->real_device); -} -symver(__ibv_get_device_guid_1_0, ibv_get_device_guid, IBVERBS_1.0); - -static int poll_cq_wrapper_1_0(struct ibv_cq_1_0 *cq, int num_entries, - struct ibv_wc *wc) -{ - return cq->context->real_context->ops.poll_cq(cq->real_cq, num_entries, wc); -} - -static int req_notify_cq_wrapper_1_0(struct ibv_cq_1_0 *cq, int sol_only) -{ - return cq->context->real_context->ops.req_notify_cq(cq->real_cq, sol_only); -} - -static int post_srq_recv_wrapper_1_0(struct ibv_srq_1_0 *srq, struct ibv_recv_wr_1_0 *wr, - struct ibv_recv_wr_1_0 **bad_wr) -{ - struct ibv_recv_wr_1_0 *w; - struct ibv_recv_wr *real_wr, *head_wr = NULL, *tail_wr = NULL, *real_bad_wr; - int ret; - - for (w = wr; w; w = w->next) { - real_wr = alloca(sizeof *real_wr); - real_wr->wr_id = w->wr_id; - real_wr->sg_list = w->sg_list; - real_wr->num_sge = w->num_sge; - real_wr->next = NULL; - if (tail_wr) - tail_wr->next = real_wr; - else - head_wr = real_wr; - - tail_wr = real_wr; - } - - ret = srq->context->real_context->ops.post_srq_recv(srq->real_srq, head_wr, - &real_bad_wr); - - if (ret) { - for (real_wr = head_wr, w = wr; - real_wr; - real_wr = real_wr->next, w = w->next) - if (real_wr == real_bad_wr) { - *bad_wr = w; - break; - } - } - - return ret; -} - -static int post_send_wrapper_1_0(struct ibv_qp_1_0 *qp, struct ibv_send_wr_1_0 *wr, - struct ibv_send_wr_1_0 **bad_wr) -{ - struct ibv_send_wr_1_0 *w; - struct ibv_send_wr *real_wr, *head_wr = NULL, *tail_wr = NULL, *real_bad_wr; - int is_ud = qp->qp_type == IBV_QPT_UD; - int ret; - - for (w = wr; w; w = w->next) { - real_wr = alloca(sizeof *real_wr); - real_wr->wr_id = w->wr_id; - real_wr->next = NULL; - -#define TEST_SIZE_2_POINT(f1, f2) \ - ((offsetof(struct ibv_send_wr, f1) - offsetof(struct ibv_send_wr, f2)) \ - == offsetof(struct ibv_send_wr_1_0, f1) - offsetof(struct ibv_send_wr_1_0, f2)) -#define TEST_SIZE_TO_END(f1) \ - ((sizeof(struct ibv_send_wr) - offsetof(struct ibv_send_wr, f1)) == \ - (sizeof(struct ibv_send_wr_1_0) - offsetof(struct ibv_send_wr_1_0, f1))) - - if (TEST_SIZE_TO_END (sg_list)) - memcpy(&real_wr->sg_list, &w->sg_list, sizeof *real_wr - - offsetof(struct ibv_send_wr, sg_list)); - else if (TEST_SIZE_2_POINT (imm_data, sg_list) && - TEST_SIZE_TO_END (wr)) { - /* we have alignment up to wr, but padding between - * imm_data and wr, and we know wr itself is the - * same size */ - memcpy(&real_wr->sg_list, &w->sg_list, - offsetof(struct ibv_send_wr, imm_data) - - offsetof(struct ibv_send_wr, sg_list) + - sizeof real_wr->imm_data); - memcpy(&real_wr->wr, &w->wr, sizeof real_wr->wr); - } else { - real_wr->sg_list = w->sg_list; - real_wr->num_sge = w->num_sge; - real_wr->opcode = w->opcode; - real_wr->send_flags = w->send_flags; - real_wr->imm_data = w->imm_data; - if (TEST_SIZE_TO_END (wr)) - memcpy(&real_wr->wr, &w->wr, - sizeof real_wr->wr); - else { - real_wr->wr.atomic.remote_addr = - w->wr.atomic.remote_addr; - real_wr->wr.atomic.compare_add = - w->wr.atomic.compare_add; - real_wr->wr.atomic.swap = - w->wr.atomic.swap; - real_wr->wr.atomic.rkey = - w->wr.atomic.rkey; - } - } - - if (is_ud) - real_wr->wr.ud.ah = w->wr.ud.ah->real_ah; - - if (tail_wr) - tail_wr->next = real_wr; - else - head_wr = real_wr; - - tail_wr = real_wr; - } - - ret = qp->context->real_context->ops.post_send(qp->real_qp, head_wr, - &real_bad_wr); - - if (ret) { - for (real_wr = head_wr, w = wr; - real_wr; - real_wr = real_wr->next, w = w->next) - if (real_wr == real_bad_wr) { - *bad_wr = w; - break; - } - } - - return ret; -} - -static int post_recv_wrapper_1_0(struct ibv_qp_1_0 *qp, struct ibv_recv_wr_1_0 *wr, - struct ibv_recv_wr_1_0 **bad_wr) -{ - struct ibv_recv_wr_1_0 *w; - struct ibv_recv_wr *real_wr, *head_wr = NULL, *tail_wr = NULL, *real_bad_wr; - int ret; - - for (w = wr; w; w = w->next) { - real_wr = alloca(sizeof *real_wr); - real_wr->wr_id = w->wr_id; - real_wr->sg_list = w->sg_list; - real_wr->num_sge = w->num_sge; - real_wr->next = NULL; - if (tail_wr) - tail_wr->next = real_wr; - else - head_wr = real_wr; - - tail_wr = real_wr; - } - - ret = qp->context->real_context->ops.post_recv(qp->real_qp, head_wr, - &real_bad_wr); - - if (ret) { - for (real_wr = head_wr, w = wr; - real_wr; - real_wr = real_wr->next, w = w->next) - if (real_wr == real_bad_wr) { - *bad_wr = w; - break; - } - } - - return ret; -} - -struct ibv_context_1_0 *__ibv_open_device_1_0(struct ibv_device_1_0 *device) -{ - struct ibv_context *real_ctx; - struct ibv_context_1_0 *ctx; - - ctx = malloc(sizeof *ctx); - if (!ctx) - return NULL; - - real_ctx = ibv_open_device(device->real_device); - if (!real_ctx) { - free(ctx); - return NULL; - } - - ctx->device = device; - ctx->real_context = real_ctx; - - ctx->ops.poll_cq = poll_cq_wrapper_1_0; - ctx->ops.req_notify_cq = req_notify_cq_wrapper_1_0; - ctx->ops.post_send = post_send_wrapper_1_0; - ctx->ops.post_recv = post_recv_wrapper_1_0; - ctx->ops.post_srq_recv = post_srq_recv_wrapper_1_0; - - return ctx; -} -symver(__ibv_open_device_1_0, ibv_open_device, IBVERBS_1.0); - -int __ibv_close_device_1_0(struct ibv_context_1_0 *context) -{ - int ret; - - ret = ibv_close_device(context->real_context); - if (ret) - return ret; - - free(context); - return 0; -} -symver(__ibv_close_device_1_0, ibv_close_device, IBVERBS_1.0); - -int __ibv_get_async_event_1_0(struct ibv_context_1_0 *context, - struct ibv_async_event *event) -{ - int ret; - - ret = ibv_get_async_event(context->real_context, event); - if (ret) - return ret; - - switch (event->event_type) { - case IBV_EVENT_CQ_ERR: - event->element.cq = event->element.cq->cq_context; - break; - - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_COMM_EST: - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_QP_LAST_WQE_REACHED: - event->element.qp = event->element.qp->qp_context; - break; - - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - event->element.srq = event->element.srq->srq_context; - break; - - default: - break; - } - - return ret; -} -symver(__ibv_get_async_event_1_0, ibv_get_async_event, IBVERBS_1.0); - -void __ibv_ack_async_event_1_0(struct ibv_async_event *event) -{ - struct ibv_async_event real_event = *event; - - switch (event->event_type) { - case IBV_EVENT_CQ_ERR: - real_event.element.cq = - ((struct ibv_cq_1_0 *) event->element.cq)->real_cq; - break; - - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_COMM_EST: - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_QP_LAST_WQE_REACHED: - real_event.element.qp = - ((struct ibv_qp_1_0 *) event->element.qp)->real_qp; - break; - - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - real_event.element.srq = - ((struct ibv_srq_1_0 *) event->element.srq)->real_srq; - break; - - default: - break; - } - - ibv_ack_async_event(&real_event); -} -symver(__ibv_ack_async_event_1_0, ibv_ack_async_event, IBVERBS_1.0); - -int __ibv_query_device_1_0(struct ibv_context_1_0 *context, - struct ibv_device_attr *device_attr) -{ - return ibv_query_device(context->real_context, device_attr); -} -symver(__ibv_query_device_1_0, ibv_query_device, IBVERBS_1.0); - -int __ibv_query_port_1_0(struct ibv_context_1_0 *context, uint8_t port_num, - struct ibv_port_attr *port_attr) -{ - return ibv_query_port(context->real_context, port_num, port_attr); -} -symver(__ibv_query_port_1_0, ibv_query_port, IBVERBS_1.0); - -int __ibv_query_gid_1_0(struct ibv_context_1_0 *context, uint8_t port_num, - int index, union ibv_gid *gid) -{ - return ibv_query_gid(context->real_context, port_num, index, gid); -} -symver(__ibv_query_gid_1_0, ibv_query_gid, IBVERBS_1.0); - -int __ibv_query_pkey_1_0(struct ibv_context_1_0 *context, uint8_t port_num, - int index, uint16_t *pkey) -{ - return ibv_query_pkey(context->real_context, port_num, index, pkey); -} -symver(__ibv_query_pkey_1_0, ibv_query_pkey, IBVERBS_1.0); - -struct ibv_pd_1_0 *__ibv_alloc_pd_1_0(struct ibv_context_1_0 *context) -{ - struct ibv_pd *real_pd; - struct ibv_pd_1_0 *pd; - - pd = malloc(sizeof *pd); - if (!pd) - return NULL; - - real_pd = ibv_alloc_pd(context->real_context); - if (!real_pd) { - free(pd); - return NULL; - } - - pd->context = context; - pd->real_pd = real_pd; - - return pd; -} -symver(__ibv_alloc_pd_1_0, ibv_alloc_pd, IBVERBS_1.0); - -int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd) -{ - int ret; - - ret = ibv_dealloc_pd(pd->real_pd); - if (ret) - return ret; - - free(pd); - return 0; -} -symver(__ibv_dealloc_pd_1_0, ibv_dealloc_pd, IBVERBS_1.0); - -struct ibv_mr_1_0 *__ibv_reg_mr_1_0(struct ibv_pd_1_0 *pd, void *addr, - size_t length, int access) -{ - struct ibv_mr *real_mr; - struct ibv_mr_1_0 *mr; - - mr = malloc(sizeof *mr); - if (!mr) - return NULL; - - real_mr = ibv_reg_mr(pd->real_pd, addr, length, access); - if (!real_mr) { - free(mr); - return NULL; - } - - mr->context = pd->context; - mr->pd = pd; - mr->lkey = real_mr->lkey; - mr->rkey = real_mr->rkey; - mr->real_mr = real_mr; - - return mr; -} -symver(__ibv_reg_mr_1_0, ibv_reg_mr, IBVERBS_1.0); - -int __ibv_dereg_mr_1_0(struct ibv_mr_1_0 *mr) -{ - int ret; - - ret = ibv_dereg_mr(mr->real_mr); - if (ret) - return ret; - - free(mr); - return 0; -} -symver(__ibv_dereg_mr_1_0, ibv_dereg_mr, IBVERBS_1.0); - -struct ibv_cq_1_0 *__ibv_create_cq_1_0(struct ibv_context_1_0 *context, int cqe, - void *cq_context, - struct ibv_comp_channel *channel, - int comp_vector) -{ - struct ibv_cq *real_cq; - struct ibv_cq_1_0 *cq; - - cq = malloc(sizeof *cq); - if (!cq) - return NULL; - - real_cq = ibv_create_cq(context->real_context, cqe, cq_context, - channel, comp_vector); - if (!real_cq) { - free(cq); - return NULL; - } - - cq->context = context; - cq->cq_context = cq_context; - cq->cqe = cqe; - cq->real_cq = real_cq; - - real_cq->cq_context = cq; - - return cq; -} -symver(__ibv_create_cq_1_0, ibv_create_cq, IBVERBS_1.0); - -int __ibv_resize_cq_1_0(struct ibv_cq_1_0 *cq, int cqe) -{ - return ibv_resize_cq(cq->real_cq, cqe); -} -symver(__ibv_resize_cq_1_0, ibv_resize_cq, IBVERBS_1.0); - -int __ibv_destroy_cq_1_0(struct ibv_cq_1_0 *cq) -{ - int ret; - - ret = ibv_destroy_cq(cq->real_cq); - if (ret) - return ret; - - free(cq); - return 0; -} -symver(__ibv_destroy_cq_1_0, ibv_destroy_cq, IBVERBS_1.0); - -int __ibv_get_cq_event_1_0(struct ibv_comp_channel *channel, - struct ibv_cq_1_0 **cq, void **cq_context) -{ - struct ibv_cq *real_cq; - void *cq_ptr; - int ret; - - ret = ibv_get_cq_event(channel, &real_cq, &cq_ptr); - if (ret) - return ret; - - *cq = cq_ptr; - *cq_context = (*cq)->cq_context; - - return 0; -} -symver(__ibv_get_cq_event_1_0, ibv_get_cq_event, IBVERBS_1.0); - -void __ibv_ack_cq_events_1_0(struct ibv_cq_1_0 *cq, unsigned int nevents) -{ - ibv_ack_cq_events(cq->real_cq, nevents); -} -symver(__ibv_ack_cq_events_1_0, ibv_ack_cq_events, IBVERBS_1.0); - -struct ibv_srq_1_0 *__ibv_create_srq_1_0(struct ibv_pd_1_0 *pd, - struct ibv_srq_init_attr *srq_init_attr) -{ - struct ibv_srq *real_srq; - struct ibv_srq_1_0 *srq; - - srq = malloc(sizeof *srq); - if (!srq) - return NULL; - - real_srq = ibv_create_srq(pd->real_pd, srq_init_attr); - if (!real_srq) { - free(srq); - return NULL; - } - - srq->context = pd->context; - srq->srq_context = srq_init_attr->srq_context; - srq->pd = pd; - srq->real_srq = real_srq; - - real_srq->srq_context = srq; - - return srq; -} -symver(__ibv_create_srq_1_0, ibv_create_srq, IBVERBS_1.0); - -int __ibv_modify_srq_1_0(struct ibv_srq_1_0 *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask) -{ - return ibv_modify_srq(srq->real_srq, srq_attr, srq_attr_mask); -} -symver(__ibv_modify_srq_1_0, ibv_modify_srq, IBVERBS_1.0); - -int __ibv_query_srq_1_0(struct ibv_srq_1_0 *srq, struct ibv_srq_attr *srq_attr) -{ - return ibv_query_srq(srq->real_srq, srq_attr); -} -symver(__ibv_query_srq_1_0, ibv_query_srq, IBVERBS_1.0); - -int __ibv_destroy_srq_1_0(struct ibv_srq_1_0 *srq) -{ - int ret; - - ret = ibv_destroy_srq(srq->real_srq); - if (ret) - return ret; - - free(srq); - return 0; -} -symver(__ibv_destroy_srq_1_0, ibv_destroy_srq, IBVERBS_1.0); - -struct ibv_qp_1_0 *__ibv_create_qp_1_0(struct ibv_pd_1_0 *pd, - struct ibv_qp_init_attr_1_0 *qp_init_attr) -{ - struct ibv_qp *real_qp; - struct ibv_qp_1_0 *qp; - struct ibv_qp_init_attr real_init_attr; - - qp = malloc(sizeof *qp); - if (!qp) - return NULL; - - real_init_attr.qp_context = qp_init_attr->qp_context; - real_init_attr.send_cq = qp_init_attr->send_cq->real_cq; - real_init_attr.recv_cq = qp_init_attr->recv_cq->real_cq; - real_init_attr.srq = qp_init_attr->srq ? - qp_init_attr->srq->real_srq : NULL; - real_init_attr.cap = qp_init_attr->cap; - real_init_attr.qp_type = qp_init_attr->qp_type; - real_init_attr.sq_sig_all = qp_init_attr->sq_sig_all; - - real_qp = ibv_create_qp(pd->real_pd, &real_init_attr); - if (!real_qp) { - free(qp); - return NULL; - } - - qp->context = pd->context; - qp->qp_context = qp_init_attr->qp_context; - qp->pd = pd; - qp->send_cq = qp_init_attr->send_cq; - qp->recv_cq = qp_init_attr->recv_cq; - qp->srq = qp_init_attr->srq; - qp->qp_type = qp_init_attr->qp_type; - qp->qp_num = real_qp->qp_num; - qp->real_qp = real_qp; - - qp_init_attr->cap = real_init_attr.cap; - - real_qp->qp_context = qp; - - return qp; -} -symver(__ibv_create_qp_1_0, ibv_create_qp, IBVERBS_1.0); - -int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr_1_0 *init_attr) -{ - struct ibv_qp_init_attr real_init_attr; - int ret; - - ret = ibv_query_qp(qp->real_qp, attr, attr_mask, &real_init_attr); - if (ret) - return ret; - - init_attr->qp_context = qp->qp_context; - init_attr->send_cq = real_init_attr.send_cq->cq_context; - init_attr->recv_cq = real_init_attr.recv_cq->cq_context; - init_attr->srq = real_init_attr.srq->srq_context; - init_attr->qp_type = real_init_attr.qp_type; - init_attr->cap = real_init_attr.cap; - init_attr->sq_sig_all = real_init_attr.sq_sig_all; - - return 0; -} -symver(__ibv_query_qp_1_0, ibv_query_qp, IBVERBS_1.0); - -int __ibv_modify_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - int attr_mask) -{ - return ibv_modify_qp(qp->real_qp, attr, attr_mask); -} -symver(__ibv_modify_qp_1_0, ibv_modify_qp, IBVERBS_1.0); - -int __ibv_destroy_qp_1_0(struct ibv_qp_1_0 *qp) -{ - int ret; - - ret = ibv_destroy_qp(qp->real_qp); - if (ret) - return ret; - - free(qp); - return 0; -} -symver(__ibv_destroy_qp_1_0, ibv_destroy_qp, IBVERBS_1.0); - -struct ibv_ah_1_0 *__ibv_create_ah_1_0(struct ibv_pd_1_0 *pd, - struct ibv_ah_attr *attr) -{ - struct ibv_ah *real_ah; - struct ibv_ah_1_0 *ah; - - ah = malloc(sizeof *ah); - if (!ah) - return NULL; - - real_ah = ibv_create_ah(pd->real_pd, attr); - if (!real_ah) { - free(ah); - return NULL; - } - - ah->context = pd->context; - ah->pd = pd; - ah->real_ah = real_ah; - - return ah; -} -symver(__ibv_create_ah_1_0, ibv_create_ah, IBVERBS_1.0); - -int __ibv_destroy_ah_1_0(struct ibv_ah_1_0 *ah) -{ - int ret; - - ret = ibv_destroy_ah(ah->real_ah); - if (ret) - return ret; - - free(ah); - return 0; -} -symver(__ibv_destroy_ah_1_0, ibv_destroy_ah, IBVERBS_1.0); - -int __ibv_attach_mcast_1_0(struct ibv_qp_1_0 *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_attach_mcast(qp->real_qp, gid, lid); -} -symver(__ibv_attach_mcast_1_0, ibv_attach_mcast, IBVERBS_1.0); - -int __ibv_detach_mcast_1_0(struct ibv_qp_1_0 *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_detach_mcast(qp->real_qp, gid, lid); -} -symver(__ibv_detach_mcast_1_0, ibv_detach_mcast, IBVERBS_1.0); diff --git a/src/device.c b/src/device.c deleted file mode 100644 index 82d928a..0000000 --- a/src/device.c +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <netinet/in.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <unistd.h> -#include <stdlib.h> -#include <alloca.h> -#include <errno.h> - -#include <infiniband/arch.h> - -#include "ibverbs.h" - -static pthread_once_t device_list_once = PTHREAD_ONCE_INIT; -static int num_devices; -static struct ibv_device **device_list; - -static void count_devices(void) -{ - num_devices = ibverbs_init(&device_list); -} - -struct ibv_device **__ibv_get_device_list(int *num) -{ - struct ibv_device **l; - int i; - - if (num) - *num = 0; - - pthread_once(&device_list_once, count_devices); - - if (num_devices < 0) { - errno = -num_devices; - return NULL; - } - - l = calloc(num_devices + 1, sizeof (struct ibv_device *)); - if (!l) { - errno = ENOMEM; - return NULL; - } - - for (i = 0; i < num_devices; ++i) - l[i] = device_list[i]; - if (num) - *num = num_devices; - - return l; -} -default_symver(__ibv_get_device_list, ibv_get_device_list); - -void __ibv_free_device_list(struct ibv_device **list) -{ - free(list); -} -default_symver(__ibv_free_device_list, ibv_free_device_list); - -const char *__ibv_get_device_name(struct ibv_device *device) -{ - return device->name; -} -default_symver(__ibv_get_device_name, ibv_get_device_name); - -uint64_t __ibv_get_device_guid(struct ibv_device *device) -{ - char attr[24]; - uint64_t guid = 0; - uint16_t parts[4]; - int i; - - if (ibv_read_sysfs_file(device->ibdev_path, "node_guid", - attr, sizeof attr) < 0) - return 0; - - if (sscanf(attr, "%hx:%hx:%hx:%hx", - parts, parts + 1, parts + 2, parts + 3) != 4) - return 0; - - for (i = 0; i < 4; ++i) - guid = (guid << 16) | parts[i]; - - return htonll(guid); -} -default_symver(__ibv_get_device_guid, ibv_get_device_guid); - -struct ibv_cq_ex *__lib_ibv_create_cq_ex(struct ibv_context *context, - struct ibv_cq_init_attr_ex *cq_attr) -{ - struct verbs_context *vctx = verbs_get_ctx(context); - struct ibv_cq_ex *cq; - - if (cq_attr->wc_flags & ~IBV_CREATE_CQ_SUP_WC_FLAGS) { - errno = EOPNOTSUPP; - return NULL; - } - - pthread_mutex_lock(&context->mutex); - - cq = vctx->priv->create_cq_ex(context, cq_attr); - - if (cq) { - cq->context = context; - cq->channel = cq_attr->channel; - if (cq->channel) - ++cq->channel->refcnt; - cq->cq_context = cq_attr->cq_context; - cq->comp_events_completed = 0; - cq->async_events_completed = 0; - pthread_mutex_init(&cq->mutex, NULL); - pthread_cond_init(&cq->cond, NULL); - } - - pthread_mutex_unlock(&context->mutex); - - return cq; -} - -struct ibv_context *__ibv_open_device(struct ibv_device *device) -{ - struct verbs_device *verbs_device = verbs_get_device(device); - char *devpath; - int cmd_fd, ret; - struct ibv_context *context; - struct verbs_context *context_ex; - - if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0) - return NULL; - - /* - * We'll only be doing writes, but we need O_RDWR in case the - * provider needs to mmap() the file. - */ - cmd_fd = open(devpath, O_RDWR | O_CLOEXEC); - free(devpath); - - if (cmd_fd < 0) - return NULL; - - if (!verbs_device) { - context = device->ops.alloc_context(device, cmd_fd); - if (!context) - goto err; - } else { - struct verbs_ex_private *priv; - - /* Library now allocates the context */ - context_ex = calloc(1, sizeof(*context_ex) + - verbs_device->size_of_context); - if (!context_ex) { - errno = ENOMEM; - goto err; - } - - priv = calloc(1, sizeof(*priv)); - if (!priv) { - errno = ENOMEM; - free(context_ex); - goto err; - } - - context_ex->priv = priv; - context_ex->context.abi_compat = __VERBS_ABI_IS_EXTENDED; - context_ex->sz = sizeof(*context_ex); - - context = &context_ex->context; - ret = verbs_device->init_context(verbs_device, context, cmd_fd); - if (ret) - goto verbs_err; - /* - * In order to maintain backward/forward binary compatibility - * with apps compiled against libibverbs-1.1.8 that use the - * flow steering addition, we need to set the two - * ABI_placeholder entries to match the driver set flow - * entries. This is because apps compiled against - * libibverbs-1.1.8 use an inline ibv_create_flow and - * ibv_destroy_flow function that looks in the placeholder - * spots for the proper entry points. For apps compiled - * against libibverbs-1.1.9 and later, the inline functions - * will be looking in the right place. - */ - context_ex->ABI_placeholder1 = (void (*)(void)) context_ex->ibv_create_flow; - context_ex->ABI_placeholder2 = (void (*)(void)) context_ex->ibv_destroy_flow; - - if (context_ex->create_cq_ex) { - priv->create_cq_ex = context_ex->create_cq_ex; - context_ex->create_cq_ex = __lib_ibv_create_cq_ex; - } - } - - context->device = device; - context->cmd_fd = cmd_fd; - pthread_mutex_init(&context->mutex, NULL); - - return context; - -verbs_err: - free(context_ex->priv); - free(context_ex); -err: - close(cmd_fd); - return NULL; -} -default_symver(__ibv_open_device, ibv_open_device); - -int __ibv_close_device(struct ibv_context *context) -{ - int async_fd = context->async_fd; - int cmd_fd = context->cmd_fd; - int cq_fd = -1; - struct verbs_context *context_ex; - - context_ex = verbs_get_ctx(context); - if (context_ex) { - struct verbs_device *verbs_device = verbs_get_device(context->device); - verbs_device->uninit_context(verbs_device, context); - free(context_ex->priv); - free(context_ex); - } else { - context->device->ops.free_context(context); - } - - close(async_fd); - close(cmd_fd); - if (abi_ver <= 2) - close(cq_fd); - - return 0; -} -default_symver(__ibv_close_device, ibv_close_device); - -int __ibv_get_async_event(struct ibv_context *context, - struct ibv_async_event *event) -{ - struct ibv_kern_async_event ev; - - if (read(context->async_fd, &ev, sizeof ev) != sizeof ev) - return -1; - - event->event_type = ev.event_type; - - switch (event->event_type) { - case IBV_EVENT_CQ_ERR: - event->element.cq = (void *) (uintptr_t) ev.element; - break; - - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_COMM_EST: - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_QP_LAST_WQE_REACHED: - event->element.qp = (void *) (uintptr_t) ev.element; - break; - - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - event->element.srq = (void *) (uintptr_t) ev.element; - break; - - case IBV_EVENT_WQ_FATAL: - event->element.wq = (void *) (uintptr_t) ev.element; - break; - default: - event->element.port_num = ev.element; - break; - } - - if (context->ops.async_event) - context->ops.async_event(event); - - return 0; -} -default_symver(__ibv_get_async_event, ibv_get_async_event); - -void __ibv_ack_async_event(struct ibv_async_event *event) -{ - switch (event->event_type) { - case IBV_EVENT_CQ_ERR: - { - struct ibv_cq *cq = event->element.cq; - - pthread_mutex_lock(&cq->mutex); - ++cq->async_events_completed; - pthread_cond_signal(&cq->cond); - pthread_mutex_unlock(&cq->mutex); - - return; - } - - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_COMM_EST: - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_QP_LAST_WQE_REACHED: - { - struct ibv_qp *qp = event->element.qp; - - pthread_mutex_lock(&qp->mutex); - ++qp->events_completed; - pthread_cond_signal(&qp->cond); - pthread_mutex_unlock(&qp->mutex); - - return; - } - - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - { - struct ibv_srq *srq = event->element.srq; - - pthread_mutex_lock(&srq->mutex); - ++srq->events_completed; - pthread_cond_signal(&srq->cond); - pthread_mutex_unlock(&srq->mutex); - - return; - } - - case IBV_EVENT_WQ_FATAL: - { - struct ibv_wq *wq = event->element.wq; - - pthread_mutex_lock(&wq->mutex); - ++wq->events_completed; - pthread_cond_signal(&wq->cond); - pthread_mutex_unlock(&wq->mutex); - - return; - } - - default: - return; - } -} -default_symver(__ibv_ack_async_event, ibv_ack_async_event); diff --git a/src/enum_strs.c b/src/enum_strs.c deleted file mode 100644 index 93ffe32..0000000 --- a/src/enum_strs.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2008 Lawrence Livermore National Laboratory - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include <infiniband/verbs.h> - -const char *ibv_node_type_str(enum ibv_node_type node_type) -{ - static const char *const node_type_str[] = { - [IBV_NODE_CA] = "InfiniBand channel adapter", - [IBV_NODE_SWITCH] = "InfiniBand switch", - [IBV_NODE_ROUTER] = "InfiniBand router", - [IBV_NODE_RNIC] = "iWARP NIC", - [IBV_NODE_USNIC] = "usNIC", - [IBV_NODE_USNIC_UDP] = "usNIC UDP", - }; - - if (node_type < IBV_NODE_CA || node_type > IBV_NODE_USNIC_UDP) - return "unknown"; - - return node_type_str[node_type]; -} - -const char *ibv_port_state_str(enum ibv_port_state port_state) -{ - static const char *const port_state_str[] = { - [IBV_PORT_NOP] = "no state change (NOP)", - [IBV_PORT_DOWN] = "down", - [IBV_PORT_INIT] = "init", - [IBV_PORT_ARMED] = "armed", - [IBV_PORT_ACTIVE] = "active", - [IBV_PORT_ACTIVE_DEFER] = "active defer" - }; - - if (port_state < IBV_PORT_NOP || port_state > IBV_PORT_ACTIVE_DEFER) - return "unknown"; - - return port_state_str[port_state]; -} - -const char *ibv_event_type_str(enum ibv_event_type event) -{ - static const char *const event_type_str[] = { - [IBV_EVENT_CQ_ERR] = "CQ error", - [IBV_EVENT_QP_FATAL] = "local work queue catastrophic error", - [IBV_EVENT_QP_REQ_ERR] = "invalid request local work queue error", - [IBV_EVENT_QP_ACCESS_ERR] = "local access violation work queue error", - [IBV_EVENT_COMM_EST] = "communication established", - [IBV_EVENT_SQ_DRAINED] = "send queue drained", - [IBV_EVENT_PATH_MIG] = "path migrated", - [IBV_EVENT_PATH_MIG_ERR] = "path migration request error", - [IBV_EVENT_DEVICE_FATAL] = "local catastrophic error", - [IBV_EVENT_PORT_ACTIVE] = "port active", - [IBV_EVENT_PORT_ERR] = "port error", - [IBV_EVENT_LID_CHANGE] = "LID change", - [IBV_EVENT_PKEY_CHANGE] = "P_Key change", - [IBV_EVENT_SM_CHANGE] = "SM change", - [IBV_EVENT_SRQ_ERR] = "SRQ catastrophic error", - [IBV_EVENT_SRQ_LIMIT_REACHED] = "SRQ limit reached", - [IBV_EVENT_QP_LAST_WQE_REACHED] = "last WQE reached", - [IBV_EVENT_CLIENT_REREGISTER] = "client reregistration", - [IBV_EVENT_GID_CHANGE] = "GID table change" - }; - - if (event < IBV_EVENT_CQ_ERR || event > IBV_EVENT_GID_CHANGE) - return "unknown"; - - return event_type_str[event]; -} - -const char *ibv_wc_status_str(enum ibv_wc_status status) -{ - static const char *const wc_status_str[] = { - [IBV_WC_SUCCESS] = "success", - [IBV_WC_LOC_LEN_ERR] = "local length error", - [IBV_WC_LOC_QP_OP_ERR] = "local QP operation error", - [IBV_WC_LOC_EEC_OP_ERR] = "local EE context operation error", - [IBV_WC_LOC_PROT_ERR] = "local protection error", - [IBV_WC_WR_FLUSH_ERR] = "Work Request Flushed Error", - [IBV_WC_MW_BIND_ERR] = "memory management operation error", - [IBV_WC_BAD_RESP_ERR] = "bad response error", - [IBV_WC_LOC_ACCESS_ERR] = "local access error", - [IBV_WC_REM_INV_REQ_ERR] = "remote invalid request error", - [IBV_WC_REM_ACCESS_ERR] = "remote access error", - [IBV_WC_REM_OP_ERR] = "remote operation error", - [IBV_WC_RETRY_EXC_ERR] = "transport retry counter exceeded", - [IBV_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded", - [IBV_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error", - [IBV_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request", - [IBV_WC_REM_ABORT_ERR] = "aborted error", - [IBV_WC_INV_EECN_ERR] = "invalid EE context number", - [IBV_WC_INV_EEC_STATE_ERR] = "invalid EE context state", - [IBV_WC_FATAL_ERR] = "fatal error", - [IBV_WC_RESP_TIMEOUT_ERR] = "response timeout error", - [IBV_WC_GENERAL_ERR] = "general error" - }; - - if (status < IBV_WC_SUCCESS || status > IBV_WC_GENERAL_ERR) - return "unknown"; - - return wc_status_str[status]; -} diff --git a/src/ibverbs.h b/src/ibverbs.h deleted file mode 100644 index 062a490..0000000 --- a/src/ibverbs.h +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef IB_VERBS_H -#define IB_VERBS_H - -#include <pthread.h> - -#include <infiniband/driver.h> - -#ifdef HAVE_VALGRIND_MEMCHECK_H - -# include <valgrind/memcheck.h> - -# ifndef VALGRIND_MAKE_MEM_DEFINED -# warning "Valgrind support requested, but VALGRIND_MAKE_MEM_DEFINED not available" -# endif - -#endif /* HAVE_VALGRIND_MEMCHECK_H */ - -#ifndef VALGRIND_MAKE_MEM_DEFINED -# define VALGRIND_MAKE_MEM_DEFINED(addr, len) 0 -#endif - -#define HIDDEN __attribute__((visibility ("hidden"))) - -#define INIT __attribute__((constructor)) -#define FINI __attribute__((destructor)) - -#define DEFAULT_ABI "IBVERBS_1.1" - -#ifdef HAVE_SYMVER_SUPPORT -# define symver(name, api, ver) \ - asm(".symver " #name "," #api "@" #ver) -# define default_symver(name, api) \ - asm(".symver " #name "," #api "@@" DEFAULT_ABI) -#else -# define symver(name, api, ver) -# define default_symver(name, api) \ - extern __typeof(name) api __attribute__((alias(#name))) -#endif /* HAVE_SYMVER_SUPPORT */ - -#define PFX "libibverbs: " - -struct ibv_abi_compat_v2 { - struct ibv_comp_channel channel; - pthread_mutex_t in_use; -}; - -extern HIDDEN int abi_ver; - -HIDDEN int ibverbs_init(struct ibv_device ***list); - -struct verbs_ex_private { - struct ibv_cq_ex *(*create_cq_ex)(struct ibv_context *context, - struct ibv_cq_init_attr_ex *init_attr); -}; - -#define IBV_INIT_CMD(cmd, size, opcode) \ - do { \ - if (abi_ver > 2) \ - (cmd)->command = IB_USER_VERBS_CMD_##opcode; \ - else \ - (cmd)->command = IB_USER_VERBS_CMD_##opcode##_V2; \ - (cmd)->in_words = (size) / 4; \ - (cmd)->out_words = 0; \ - } while (0) - -#define IBV_INIT_CMD_RESP(cmd, size, opcode, out, outsize) \ - do { \ - if (abi_ver > 2) \ - (cmd)->command = IB_USER_VERBS_CMD_##opcode; \ - else \ - (cmd)->command = IB_USER_VERBS_CMD_##opcode##_V2; \ - (cmd)->in_words = (size) / 4; \ - (cmd)->out_words = (outsize) / 4; \ - (cmd)->response = (uintptr_t) (out); \ - } while (0) - -#define IBV_INIT_CMD_RESP_EX_V(cmd, cmd_size, size, opcode, out, resp_size,\ - outsize) \ - do { \ - size_t c_size = cmd_size - sizeof(struct ex_hdr); \ - if (abi_ver > 2) \ - (cmd)->hdr.command = IB_USER_VERBS_CMD_##opcode; \ - else \ - (cmd)->hdr.command = \ - IB_USER_VERBS_CMD_##opcode##_V2; \ - (cmd)->hdr.in_words = ((c_size) / 8); \ - (cmd)->hdr.out_words = ((resp_size) / 8); \ - (cmd)->hdr.provider_in_words = (((size) - (cmd_size))/8);\ - (cmd)->hdr.provider_out_words = \ - (((outsize) - (resp_size)) / 8); \ - (cmd)->hdr.response = (uintptr_t) (out); \ - (cmd)->hdr.reserved = 0; \ - } while (0) - -#define IBV_INIT_CMD_RESP_EX_VCMD(cmd, cmd_size, size, opcode, out, outsize) \ - IBV_INIT_CMD_RESP_EX_V(cmd, cmd_size, size, opcode, out, \ - sizeof(*(out)), outsize) - -#define IBV_INIT_CMD_RESP_EX(cmd, size, opcode, out, outsize) \ - IBV_INIT_CMD_RESP_EX_V(cmd, sizeof(*(cmd)), size, opcode, out, \ - sizeof(*(out)), outsize) - -#define IBV_INIT_CMD_EX(cmd, size, opcode) \ - IBV_INIT_CMD_RESP_EX_V(cmd, sizeof(*(cmd)), size, opcode, NULL, 0, 0) - -#endif /* IB_VERBS_H */ diff --git a/src/init.c b/src/init.c deleted file mode 100644 index dbdd795..0000000 --- a/src/init.c +++ /dev/null @@ -1,575 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdlib.h> -#include <string.h> -#include <glob.h> -#include <stdio.h> -#include <dlfcn.h> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <sys/time.h> -#include <sys/resource.h> -#include <dirent.h> -#include <errno.h> - -#include "ibverbs.h" - -HIDDEN int abi_ver; - -struct ibv_sysfs_dev { - char sysfs_name[IBV_SYSFS_NAME_MAX]; - char ibdev_name[IBV_SYSFS_NAME_MAX]; - char sysfs_path[IBV_SYSFS_PATH_MAX]; - char ibdev_path[IBV_SYSFS_PATH_MAX]; - struct ibv_sysfs_dev *next; - int abi_ver; - int have_driver; -}; - -struct ibv_driver_name { - char *name; - struct ibv_driver_name *next; -}; - -struct ibv_driver { - const char *name; - ibv_driver_init_func init_func; - verbs_driver_init_func verbs_init_func; - struct ibv_driver *next; -}; - -static struct ibv_sysfs_dev *sysfs_dev_list; -static struct ibv_driver_name *driver_name_list; -static struct ibv_driver *head_driver, *tail_driver; - -static int find_sysfs_devs(void) -{ - char class_path[IBV_SYSFS_PATH_MAX]; - DIR *class_dir; - struct dirent *dent; - struct ibv_sysfs_dev *sysfs_dev = NULL; - char value[8]; - int ret = 0; - - snprintf(class_path, sizeof class_path, "%s/class/infiniband_verbs", - ibv_get_sysfs_path()); - - class_dir = opendir(class_path); - if (!class_dir) - return ENOSYS; - - while ((dent = readdir(class_dir))) { - struct stat buf; - - if (dent->d_name[0] == '.') - continue; - - if (!sysfs_dev) - sysfs_dev = malloc(sizeof *sysfs_dev); - if (!sysfs_dev) { - ret = ENOMEM; - goto out; - } - - snprintf(sysfs_dev->sysfs_path, sizeof sysfs_dev->sysfs_path, - "%s/%s", class_path, dent->d_name); - - if (stat(sysfs_dev->sysfs_path, &buf)) { - fprintf(stderr, PFX "Warning: couldn't stat '%s'.\n", - sysfs_dev->sysfs_path); - continue; - } - - if (!S_ISDIR(buf.st_mode)) - continue; - - snprintf(sysfs_dev->sysfs_name, sizeof sysfs_dev->sysfs_name, - "%s", dent->d_name); - - if (ibv_read_sysfs_file(sysfs_dev->sysfs_path, "ibdev", - sysfs_dev->ibdev_name, - sizeof sysfs_dev->ibdev_name) < 0) { - fprintf(stderr, PFX "Warning: no ibdev class attr for '%s'.\n", - dent->d_name); - continue; - } - - snprintf(sysfs_dev->ibdev_path, sizeof sysfs_dev->ibdev_path, - "%s/class/infiniband/%s", ibv_get_sysfs_path(), - sysfs_dev->ibdev_name); - - sysfs_dev->next = sysfs_dev_list; - sysfs_dev->have_driver = 0; - if (ibv_read_sysfs_file(sysfs_dev->sysfs_path, "abi_version", - value, sizeof value) > 0) - sysfs_dev->abi_ver = strtol(value, NULL, 10); - else - sysfs_dev->abi_ver = 0; - - sysfs_dev_list = sysfs_dev; - sysfs_dev = NULL; - } - - out: - if (sysfs_dev) - free(sysfs_dev); - - closedir(class_dir); - return ret; -} - -static void register_driver(const char *name, ibv_driver_init_func init_func, - verbs_driver_init_func verbs_init_func) -{ - struct ibv_driver *driver; - - driver = malloc(sizeof *driver); - if (!driver) { - fprintf(stderr, PFX "Warning: couldn't allocate driver for %s\n", name); - return; - } - - driver->name = name; - driver->init_func = init_func; - driver->verbs_init_func = verbs_init_func; - driver->next = NULL; - - if (tail_driver) - tail_driver->next = driver; - else - head_driver = driver; - tail_driver = driver; -} - -void ibv_register_driver(const char *name, ibv_driver_init_func init_func) -{ - register_driver(name, init_func, NULL); -} - -/* New registration symbol with same functionality - used by providers to - * validate that library supports verbs extension. - */ -void verbs_register_driver(const char *name, verbs_driver_init_func init_func) -{ - register_driver(name, NULL, init_func); -} - -static void load_driver(const char *name) -{ - char *so_name; - void *dlhandle; - -#define __IBV_QUOTE(x) #x -#define IBV_QUOTE(x) __IBV_QUOTE(x) - - if (asprintf(&so_name, - name[0] == '/' ? - "%s-" IBV_QUOTE(IBV_DEVICE_LIBRARY_EXTENSION) ".so" : - "lib%s-" IBV_QUOTE(IBV_DEVICE_LIBRARY_EXTENSION) ".so", - name) < 0) { - fprintf(stderr, PFX "Warning: couldn't load driver '%s'.\n", - name); - return; - } - - dlhandle = dlopen(so_name, RTLD_NOW); - if (!dlhandle) { - fprintf(stderr, PFX "Warning: couldn't load driver '%s': %s\n", - name, dlerror()); - goto out; - } - -out: - free(so_name); -} - -static void load_drivers(void) -{ - struct ibv_driver_name *name, *next_name; - const char *env; - char *list, *env_name; - - /* - * Only use drivers passed in through the calling user's - * environment if we're not running setuid. - */ - if (getuid() == geteuid()) { - if ((env = getenv("RDMAV_DRIVERS"))) { - list = strdupa(env); - while ((env_name = strsep(&list, ":;"))) - load_driver(env_name); - } else if ((env = getenv("IBV_DRIVERS"))) { - list = strdupa(env); - while ((env_name = strsep(&list, ":;"))) - load_driver(env_name); - } - } - - for (name = driver_name_list, next_name = name ? name->next : NULL; - name; - name = next_name, next_name = name ? name->next : NULL) { - load_driver(name->name); - free(name->name); - free(name); - } -} - -static void read_config_file(const char *path) -{ - FILE *conf; - char *line = NULL; - char *config; - char *field; - size_t buflen = 0; - ssize_t len; - - conf = fopen(path, "r" STREAM_CLOEXEC); - if (!conf) { - fprintf(stderr, PFX "Warning: couldn't read config file %s.\n", - path); - return; - } - - while ((len = getline(&line, &buflen, conf)) != -1) { - config = line + strspn(line, "\t "); - if (config[0] == '\n' || config[0] == '#') - continue; - - field = strsep(&config, "\n\t "); - - if (strcmp(field, "driver") == 0 && config != NULL) { - struct ibv_driver_name *driver_name; - - config += strspn(config, "\t "); - field = strsep(&config, "\n\t "); - - driver_name = malloc(sizeof *driver_name); - if (!driver_name) { - fprintf(stderr, PFX "Warning: couldn't allocate " - "driver name '%s'.\n", field); - continue; - } - - driver_name->name = strdup(field); - if (!driver_name->name) { - fprintf(stderr, PFX "Warning: couldn't allocate " - "driver name '%s'.\n", field); - free(driver_name); - continue; - } - - driver_name->next = driver_name_list; - driver_name_list = driver_name; - } else - fprintf(stderr, PFX "Warning: ignoring bad config directive " - "'%s' in file '%s'.\n", field, path); - } - - if (line) - free(line); - fclose(conf); -} - -static void read_config(void) -{ - DIR *conf_dir; - struct dirent *dent; - char *path; - - conf_dir = opendir(IBV_CONFIG_DIR); - if (!conf_dir) { - fprintf(stderr, PFX "Warning: couldn't open config directory '%s'.\n", - IBV_CONFIG_DIR); - return; - } - - while ((dent = readdir(conf_dir))) { - struct stat buf; - - if (asprintf(&path, "%s/%s", IBV_CONFIG_DIR, dent->d_name) < 0) { - fprintf(stderr, PFX "Warning: couldn't read config file %s/%s.\n", - IBV_CONFIG_DIR, dent->d_name); - goto out; - } - - if (stat(path, &buf)) { - fprintf(stderr, PFX "Warning: couldn't stat config file '%s'.\n", - path); - goto next; - } - - if (!S_ISREG(buf.st_mode)) - goto next; - - read_config_file(path); -next: - free(path); - } - -out: - closedir(conf_dir); -} - -static struct ibv_device *try_driver(struct ibv_driver *driver, - struct ibv_sysfs_dev *sysfs_dev) -{ - struct verbs_device *vdev; - struct ibv_device *dev; - char value[8]; - - if (driver->init_func) { - dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver); - if (!dev) - return NULL; - } else { - vdev = driver->verbs_init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver); - if (!vdev) - return NULL; - - dev = &vdev->device; - dev->ops.alloc_context = NULL; - dev->ops.free_context = NULL; - } - - if (ibv_read_sysfs_file(sysfs_dev->ibdev_path, "node_type", value, sizeof value) < 0) { - fprintf(stderr, PFX "Warning: no node_type attr under %s.\n", - sysfs_dev->ibdev_path); - dev->node_type = IBV_NODE_UNKNOWN; - } else { - dev->node_type = strtol(value, NULL, 10); - if (dev->node_type < IBV_NODE_CA || dev->node_type > IBV_NODE_USNIC_UDP) - dev->node_type = IBV_NODE_UNKNOWN; - } - - switch (dev->node_type) { - case IBV_NODE_CA: - case IBV_NODE_SWITCH: - case IBV_NODE_ROUTER: - dev->transport_type = IBV_TRANSPORT_IB; - break; - case IBV_NODE_RNIC: - dev->transport_type = IBV_TRANSPORT_IWARP; - break; - case IBV_NODE_USNIC: - dev->transport_type = IBV_TRANSPORT_USNIC; - break; - case IBV_NODE_USNIC_UDP: - dev->transport_type = IBV_TRANSPORT_USNIC_UDP; - break; - default: - dev->transport_type = IBV_TRANSPORT_UNKNOWN; - break; - } - - strcpy(dev->dev_name, sysfs_dev->sysfs_name); - strcpy(dev->dev_path, sysfs_dev->sysfs_path); - strcpy(dev->name, sysfs_dev->ibdev_name); - strcpy(dev->ibdev_path, sysfs_dev->ibdev_path); - - return dev; -} - -static struct ibv_device *try_drivers(struct ibv_sysfs_dev *sysfs_dev) -{ - struct ibv_driver *driver; - struct ibv_device *dev; - - for (driver = head_driver; driver; driver = driver->next) { - dev = try_driver(driver, sysfs_dev); - if (dev) - return dev; - } - - return NULL; -} - -static int check_abi_version(const char *path) -{ - char value[8]; - - if (ibv_read_sysfs_file(path, "class/infiniband_verbs/abi_version", - value, sizeof value) < 0) { - return ENOSYS; - } - - abi_ver = strtol(value, NULL, 10); - - if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION || - abi_ver > IB_USER_VERBS_MAX_ABI_VERSION) { - fprintf(stderr, PFX "Fatal: kernel ABI version %d " - "doesn't match library version %d.\n", - abi_ver, IB_USER_VERBS_MAX_ABI_VERSION); - return ENOSYS; - } - - return 0; -} - -static void check_memlock_limit(void) -{ - struct rlimit rlim; - - if (!geteuid()) - return; - - if (getrlimit(RLIMIT_MEMLOCK, &rlim)) { - fprintf(stderr, PFX "Warning: getrlimit(RLIMIT_MEMLOCK) failed."); - return; - } - - if (rlim.rlim_cur <= 32768) - fprintf(stderr, PFX "Warning: RLIMIT_MEMLOCK is %lu bytes.\n" - " This will severely limit memory registrations.\n", - rlim.rlim_cur); -} - -static void add_device(struct ibv_device *dev, - struct ibv_device ***dev_list, - int *num_devices, - int *list_size) -{ - struct ibv_device **new_list; - - if (*list_size <= *num_devices) { - *list_size = *list_size ? *list_size * 2 : 1; - new_list = realloc(*dev_list, *list_size * sizeof (struct ibv_device *)); - if (!new_list) - return; - *dev_list = new_list; - } - - (*dev_list)[(*num_devices)++] = dev; -} - -HIDDEN int ibverbs_init(struct ibv_device ***list) -{ - const char *sysfs_path; - struct ibv_sysfs_dev *sysfs_dev, *next_dev; - struct ibv_device *device; - int num_devices = 0; - int list_size = 0; - int statically_linked = 0; - int no_driver = 0; - int ret; - - *list = NULL; - - if (getenv("RDMAV_FORK_SAFE") || getenv("IBV_FORK_SAFE")) - if (ibv_fork_init()) - fprintf(stderr, PFX "Warning: fork()-safety requested " - "but init failed\n"); - - sysfs_path = ibv_get_sysfs_path(); - if (!sysfs_path) - return -ENOSYS; - - ret = check_abi_version(sysfs_path); - if (ret) - return -ret; - - check_memlock_limit(); - - read_config(); - - ret = find_sysfs_devs(); - if (ret) - return -ret; - - for (sysfs_dev = sysfs_dev_list; sysfs_dev; sysfs_dev = sysfs_dev->next) { - device = try_drivers(sysfs_dev); - if (device) { - add_device(device, list, &num_devices, &list_size); - sysfs_dev->have_driver = 1; - } else - no_driver = 1; - } - - if (!no_driver) - goto out; - - /* - * Check if we can dlopen() ourselves. If this fails, - * libibverbs is probably statically linked into the - * executable, and we should just give up, since trying to - * dlopen() a driver module will fail spectacularly (loading a - * driver .so will bring in dynamic copies of libibverbs and - * libdl to go along with the static copies the executable - * has, which quickly leads to a crash. - */ - { - void *hand = dlopen(NULL, RTLD_NOW); - if (!hand) { - fprintf(stderr, PFX "Warning: dlopen(NULL) failed, " - "assuming static linking.\n"); - statically_linked = 1; - goto out; - } - dlclose(hand); - } - - load_drivers(); - - for (sysfs_dev = sysfs_dev_list; sysfs_dev; sysfs_dev = sysfs_dev->next) { - if (sysfs_dev->have_driver) - continue; - - device = try_drivers(sysfs_dev); - if (device) { - add_device(device, list, &num_devices, &list_size); - sysfs_dev->have_driver = 1; - } - } - -out: - for (sysfs_dev = sysfs_dev_list, - next_dev = sysfs_dev ? sysfs_dev->next : NULL; - sysfs_dev; - sysfs_dev = next_dev, next_dev = sysfs_dev ? sysfs_dev->next : NULL) { - if (!sysfs_dev->have_driver && getenv("IBV_SHOW_WARNINGS")) { - fprintf(stderr, PFX "Warning: no userspace device-specific " - "driver found for %s\n", sysfs_dev->sysfs_path); - if (statically_linked) - fprintf(stderr, " When linking libibverbs statically, " - "driver must be statically linked too.\n"); - } - free(sysfs_dev); - } - - return num_devices; -} diff --git a/src/libibverbs.map b/src/libibverbs.map deleted file mode 100644 index 46744e5..0000000 --- a/src/libibverbs.map +++ /dev/null @@ -1,132 +0,0 @@ -IBVERBS_1.0 { - global: - ibv_get_device_list; - ibv_free_device_list; - ibv_get_device_name; - ibv_get_device_guid; - ibv_open_device; - ibv_close_device; - ibv_get_async_event; - ibv_ack_async_event; - ibv_query_device; - ibv_query_device_ex; - ibv_query_port; - ibv_query_gid; - ibv_query_pkey; - ibv_alloc_pd; - ibv_dealloc_pd; - ibv_reg_mr; - ibv_dereg_mr; - ibv_create_comp_channel; - ibv_destroy_comp_channel; - ibv_create_cq; - ibv_resize_cq; - ibv_destroy_cq; - ibv_get_cq_event; - ibv_ack_cq_events; - ibv_create_srq; - ibv_modify_srq; - ibv_query_srq; - ibv_destroy_srq; - ibv_create_qp; - ibv_query_qp; - ibv_modify_qp; - ibv_destroy_qp; - ibv_create_ah; - ibv_destroy_ah; - ibv_attach_mcast; - ibv_detach_mcast; - ibv_cmd_get_context; - ibv_cmd_query_device; - ibv_cmd_query_device_ex; - ibv_cmd_query_port; - ibv_cmd_query_gid; - ibv_cmd_query_pkey; - ibv_cmd_alloc_pd; - ibv_cmd_dealloc_pd; - ibv_cmd_reg_mr; - ibv_cmd_dereg_mr; - ibv_cmd_create_cq; - ibv_cmd_create_cq_ex; - ibv_cmd_poll_cq; - ibv_cmd_req_notify_cq; - ibv_cmd_resize_cq; - ibv_cmd_destroy_cq; - ibv_cmd_create_srq; - ibv_cmd_modify_srq; - ibv_cmd_query_srq; - ibv_cmd_destroy_srq; - ibv_cmd_create_qp; - ibv_cmd_query_qp; - ibv_cmd_modify_qp; - ibv_cmd_destroy_qp; - ibv_cmd_post_send; - ibv_cmd_post_recv; - ibv_cmd_post_srq_recv; - ibv_cmd_create_ah; - ibv_cmd_destroy_ah; - ibv_cmd_attach_mcast; - ibv_cmd_detach_mcast; - ibv_cmd_create_flow; - ibv_cmd_destroy_flow; - ibv_copy_qp_attr_from_kern; - ibv_copy_path_rec_from_kern; - ibv_copy_path_rec_to_kern; - ibv_rate_to_mult; - mult_to_ibv_rate; - ibv_get_sysfs_path; - ibv_read_sysfs_file; - - local: *; -}; - -IBVERBS_1.1 { - global: - ibv_get_device_list; - ibv_free_device_list; - ibv_get_device_name; - ibv_get_device_guid; - ibv_open_device; - ibv_close_device; - - ibv_init_ah_from_wc; - ibv_create_ah_from_wc; - ibv_copy_ah_attr_from_kern; - ibv_fork_init; - ibv_dontfork_range; - ibv_dofork_range; - ibv_register_driver; - verbs_register_driver; - - ibv_node_type_str; - ibv_port_state_str; - ibv_event_type_str; - ibv_wc_status_str; - - ibv_cmd_alloc_mw; - ibv_cmd_dealloc_mw; - - ibv_rate_to_mbps; - mbps_to_ibv_rate; - - ibv_resolve_eth_l2_from_gid; - - ibv_cmd_open_xrcd; - ibv_cmd_close_xrcd; - ibv_cmd_create_srq_ex; - ibv_cmd_create_qp_ex; - ibv_cmd_create_qp_ex2; - ibv_cmd_open_qp; - ibv_cmd_rereg_mr; - -} IBVERBS_1.0; - -IBVERBS_1.3 { - global: - ibv_cmd_create_wq; - ibv_cmd_modify_wq; - ibv_cmd_destroy_wq; - ibv_cmd_create_rwq_ind_table; - ibv_cmd_destroy_rwq_ind_table; - ibv_query_gid_type; -} IBVERBS_1.1; diff --git a/src/marshall.c b/src/marshall.c deleted file mode 100644 index 577b4b1..0000000 --- a/src/marshall.c +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <string.h> - -#include <infiniband/marshall.h> - -void ibv_copy_ah_attr_from_kern(struct ibv_ah_attr *dst, - struct ibv_kern_ah_attr *src) -{ - memcpy(dst->grh.dgid.raw, src->grh.dgid, sizeof dst->grh.dgid); - dst->grh.flow_label = src->grh.flow_label; - dst->grh.sgid_index = src->grh.sgid_index; - dst->grh.hop_limit = src->grh.hop_limit; - dst->grh.traffic_class = src->grh.traffic_class; - - dst->dlid = src->dlid; - dst->sl = src->sl; - dst->src_path_bits = src->src_path_bits; - dst->static_rate = src->static_rate; - dst->is_global = src->is_global; - dst->port_num = src->port_num; -} - -void ibv_copy_qp_attr_from_kern(struct ibv_qp_attr *dst, - struct ibv_kern_qp_attr *src) -{ - dst->cur_qp_state = src->cur_qp_state; - dst->path_mtu = src->path_mtu; - dst->path_mig_state = src->path_mig_state; - dst->qkey = src->qkey; - dst->rq_psn = src->rq_psn; - dst->sq_psn = src->sq_psn; - dst->dest_qp_num = src->dest_qp_num; - dst->qp_access_flags = src->qp_access_flags; - - dst->cap.max_send_wr = src->max_send_wr; - dst->cap.max_recv_wr = src->max_recv_wr; - dst->cap.max_send_sge = src->max_send_sge; - dst->cap.max_recv_sge = src->max_recv_sge; - dst->cap.max_inline_data = src->max_inline_data; - - ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr); - ibv_copy_ah_attr_from_kern(&dst->alt_ah_attr, &src->alt_ah_attr); - - dst->pkey_index = src->pkey_index; - dst->alt_pkey_index = src->alt_pkey_index; - dst->en_sqd_async_notify = src->en_sqd_async_notify; - dst->sq_draining = src->sq_draining; - dst->max_rd_atomic = src->max_rd_atomic; - dst->max_dest_rd_atomic = src->max_dest_rd_atomic; - dst->min_rnr_timer = src->min_rnr_timer; - dst->port_num = src->port_num; - dst->timeout = src->timeout; - dst->retry_cnt = src->retry_cnt; - dst->rnr_retry = src->rnr_retry; - dst->alt_port_num = src->alt_port_num; - dst->alt_timeout = src->alt_timeout; -} - -void ibv_copy_path_rec_from_kern(struct ibv_sa_path_rec *dst, - struct ibv_kern_path_rec *src) -{ - memcpy(dst->dgid.raw, src->dgid, sizeof dst->dgid); - memcpy(dst->sgid.raw, src->sgid, sizeof dst->sgid); - - dst->dlid = src->dlid; - dst->slid = src->slid; - dst->raw_traffic = src->raw_traffic; - dst->flow_label = src->flow_label; - dst->hop_limit = src->hop_limit; - dst->traffic_class = src->traffic_class; - dst->reversible = src->reversible; - dst->numb_path = src->numb_path; - dst->pkey = src->pkey; - dst->sl = src->sl; - dst->mtu_selector = src->mtu_selector; - dst->mtu = src->mtu; - dst->rate_selector = src->rate_selector; - dst->rate = src->rate; - dst->packet_life_time = src->packet_life_time; - dst->preference = src->preference; - dst->packet_life_time_selector = src->packet_life_time_selector; -} - -void ibv_copy_path_rec_to_kern(struct ibv_kern_path_rec *dst, - struct ibv_sa_path_rec *src) -{ - memcpy(dst->dgid, src->dgid.raw, sizeof src->dgid); - memcpy(dst->sgid, src->sgid.raw, sizeof src->sgid); - - dst->dlid = src->dlid; - dst->slid = src->slid; - dst->raw_traffic = src->raw_traffic; - dst->flow_label = src->flow_label; - dst->hop_limit = src->hop_limit; - dst->traffic_class = src->traffic_class; - dst->reversible = src->reversible; - dst->numb_path = src->numb_path; - dst->pkey = src->pkey; - dst->sl = src->sl; - dst->mtu_selector = src->mtu_selector; - dst->mtu = src->mtu; - dst->rate_selector = src->rate_selector; - dst->rate = src->rate; - dst->packet_life_time = src->packet_life_time; - dst->preference = src->preference; - dst->packet_life_time_selector = src->packet_life_time_selector; -} diff --git a/src/memory.c b/src/memory.c deleted file mode 100644 index 89509c6..0000000 --- a/src/memory.c +++ /dev/null @@ -1,717 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <errno.h> -#include <sys/mman.h> -#include <unistd.h> -#include <stdlib.h> -#include <stdint.h> -#include <stdio.h> -#include <string.h> -#include <dirent.h> -#include <limits.h> -#include <inttypes.h> - -#include "ibverbs.h" - -/* - * Most distro's headers don't have these yet. - */ -#ifndef MADV_DONTFORK -#define MADV_DONTFORK 10 -#endif - -#ifndef MADV_DOFORK -#define MADV_DOFORK 11 -#endif - -struct ibv_mem_node { - enum { - IBV_RED, - IBV_BLACK - } color; - struct ibv_mem_node *parent; - struct ibv_mem_node *left, *right; - uintptr_t start, end; - int refcnt; -}; - -static struct ibv_mem_node *mm_root; -static pthread_mutex_t mm_mutex = PTHREAD_MUTEX_INITIALIZER; -static int page_size; -static int huge_page_enabled; -static int too_late; - -static unsigned long smaps_page_size(FILE *file) -{ - int n; - unsigned long size = page_size; - char buf[1024]; - - while (fgets(buf, sizeof(buf), file) != NULL) { - if (!strstr(buf, "KernelPageSize:")) - continue; - - n = sscanf(buf, "%*s %lu", &size); - if (n < 1) - continue; - - /* page size is printed in Kb */ - size = size * 1024; - - break; - } - - return size; -} - -static unsigned long get_page_size(void *base) -{ - unsigned long ret = page_size; - pid_t pid; - FILE *file; - char buf[1024]; - - pid = getpid(); - snprintf(buf, sizeof(buf), "/proc/%d/smaps", pid); - - file = fopen(buf, "r" STREAM_CLOEXEC); - if (!file) - goto out; - - while (fgets(buf, sizeof(buf), file) != NULL) { - int n; - uintptr_t range_start, range_end; - - n = sscanf(buf, "%" SCNxPTR "-%" SCNxPTR, &range_start, &range_end); - - if (n < 2) - continue; - - if ((uintptr_t) base >= range_start && (uintptr_t) base < range_end) { - ret = smaps_page_size(file); - break; - } - } - - fclose(file); - -out: - return ret; -} - -int ibv_fork_init(void) -{ - void *tmp, *tmp_aligned; - int ret; - unsigned long size; - - if (getenv("RDMAV_HUGEPAGES_SAFE")) - huge_page_enabled = 1; - - if (mm_root) - return 0; - - if (too_late) - return EINVAL; - - page_size = sysconf(_SC_PAGESIZE); - if (page_size < 0) - return errno; - - if (posix_memalign(&tmp, page_size, page_size)) - return ENOMEM; - - if (huge_page_enabled) { - size = get_page_size(tmp); - tmp_aligned = (void *) ((uintptr_t) tmp & ~(size - 1)); - } else { - size = page_size; - tmp_aligned = tmp; - } - - ret = madvise(tmp_aligned, size, MADV_DONTFORK) || - madvise(tmp_aligned, size, MADV_DOFORK); - - free(tmp); - - if (ret) - return ENOSYS; - - mm_root = malloc(sizeof *mm_root); - if (!mm_root) - return ENOMEM; - - mm_root->parent = NULL; - mm_root->left = NULL; - mm_root->right = NULL; - mm_root->color = IBV_BLACK; - mm_root->start = 0; - mm_root->end = UINTPTR_MAX; - mm_root->refcnt = 0; - - return 0; -} - -static struct ibv_mem_node *__mm_prev(struct ibv_mem_node *node) -{ - if (node->left) { - node = node->left; - while (node->right) - node = node->right; - } else { - while (node->parent && node == node->parent->left) - node = node->parent; - - node = node->parent; - } - - return node; -} - -static struct ibv_mem_node *__mm_next(struct ibv_mem_node *node) -{ - if (node->right) { - node = node->right; - while (node->left) - node = node->left; - } else { - while (node->parent && node == node->parent->right) - node = node->parent; - - node = node->parent; - } - - return node; -} - -static void __mm_rotate_right(struct ibv_mem_node *node) -{ - struct ibv_mem_node *tmp; - - tmp = node->left; - - node->left = tmp->right; - if (node->left) - node->left->parent = node; - - if (node->parent) { - if (node->parent->right == node) - node->parent->right = tmp; - else - node->parent->left = tmp; - } else - mm_root = tmp; - - tmp->parent = node->parent; - - tmp->right = node; - node->parent = tmp; -} - -static void __mm_rotate_left(struct ibv_mem_node *node) -{ - struct ibv_mem_node *tmp; - - tmp = node->right; - - node->right = tmp->left; - if (node->right) - node->right->parent = node; - - if (node->parent) { - if (node->parent->right == node) - node->parent->right = tmp; - else - node->parent->left = tmp; - } else - mm_root = tmp; - - tmp->parent = node->parent; - - tmp->left = node; - node->parent = tmp; -} - -#if 0 -static int verify(struct ibv_mem_node *node) -{ - int hl, hr; - - if (!node) - return 1; - - hl = verify(node->left); - hr = verify(node->left); - - if (!hl || !hr) - return 0; - if (hl != hr) - return 0; - - if (node->color == IBV_RED) { - if (node->left && node->left->color != IBV_BLACK) - return 0; - if (node->right && node->right->color != IBV_BLACK) - return 0; - return hl; - } - - return hl + 1; -} -#endif - -static void __mm_add_rebalance(struct ibv_mem_node *node) -{ - struct ibv_mem_node *parent, *gp, *uncle; - - while (node->parent && node->parent->color == IBV_RED) { - parent = node->parent; - gp = node->parent->parent; - - if (parent == gp->left) { - uncle = gp->right; - - if (uncle && uncle->color == IBV_RED) { - parent->color = IBV_BLACK; - uncle->color = IBV_BLACK; - gp->color = IBV_RED; - - node = gp; - } else { - if (node == parent->right) { - __mm_rotate_left(parent); - node = parent; - parent = node->parent; - } - - parent->color = IBV_BLACK; - gp->color = IBV_RED; - - __mm_rotate_right(gp); - } - } else { - uncle = gp->left; - - if (uncle && uncle->color == IBV_RED) { - parent->color = IBV_BLACK; - uncle->color = IBV_BLACK; - gp->color = IBV_RED; - - node = gp; - } else { - if (node == parent->left) { - __mm_rotate_right(parent); - node = parent; - parent = node->parent; - } - - parent->color = IBV_BLACK; - gp->color = IBV_RED; - - __mm_rotate_left(gp); - } - } - } - - mm_root->color = IBV_BLACK; -} - -static void __mm_add(struct ibv_mem_node *new) -{ - struct ibv_mem_node *node, *parent = NULL; - - node = mm_root; - while (node) { - parent = node; - if (node->start < new->start) - node = node->right; - else - node = node->left; - } - - if (parent->start < new->start) - parent->right = new; - else - parent->left = new; - - new->parent = parent; - new->left = NULL; - new->right = NULL; - - new->color = IBV_RED; - __mm_add_rebalance(new); -} - -static void __mm_remove(struct ibv_mem_node *node) -{ - struct ibv_mem_node *child, *parent, *sib, *tmp; - int nodecol; - - if (node->left && node->right) { - tmp = node->left; - while (tmp->right) - tmp = tmp->right; - - nodecol = tmp->color; - child = tmp->left; - tmp->color = node->color; - - if (tmp->parent != node) { - parent = tmp->parent; - parent->right = tmp->left; - if (tmp->left) - tmp->left->parent = parent; - - tmp->left = node->left; - node->left->parent = tmp; - } else - parent = tmp; - - tmp->right = node->right; - node->right->parent = tmp; - - tmp->parent = node->parent; - if (node->parent) { - if (node->parent->left == node) - node->parent->left = tmp; - else - node->parent->right = tmp; - } else - mm_root = tmp; - } else { - nodecol = node->color; - - child = node->left ? node->left : node->right; - parent = node->parent; - - if (child) - child->parent = parent; - if (parent) { - if (parent->left == node) - parent->left = child; - else - parent->right = child; - } else - mm_root = child; - } - - free(node); - - if (nodecol == IBV_RED) - return; - - while ((!child || child->color == IBV_BLACK) && child != mm_root) { - if (parent->left == child) { - sib = parent->right; - - if (sib->color == IBV_RED) { - parent->color = IBV_RED; - sib->color = IBV_BLACK; - __mm_rotate_left(parent); - sib = parent->right; - } - - if ((!sib->left || sib->left->color == IBV_BLACK) && - (!sib->right || sib->right->color == IBV_BLACK)) { - sib->color = IBV_RED; - child = parent; - parent = child->parent; - } else { - if (!sib->right || sib->right->color == IBV_BLACK) { - if (sib->left) - sib->left->color = IBV_BLACK; - sib->color = IBV_RED; - __mm_rotate_right(sib); - sib = parent->right; - } - - sib->color = parent->color; - parent->color = IBV_BLACK; - if (sib->right) - sib->right->color = IBV_BLACK; - __mm_rotate_left(parent); - child = mm_root; - break; - } - } else { - sib = parent->left; - - if (sib->color == IBV_RED) { - parent->color = IBV_RED; - sib->color = IBV_BLACK; - __mm_rotate_right(parent); - sib = parent->left; - } - - if ((!sib->left || sib->left->color == IBV_BLACK) && - (!sib->right || sib->right->color == IBV_BLACK)) { - sib->color = IBV_RED; - child = parent; - parent = child->parent; - } else { - if (!sib->left || sib->left->color == IBV_BLACK) { - if (sib->right) - sib->right->color = IBV_BLACK; - sib->color = IBV_RED; - __mm_rotate_left(sib); - sib = parent->left; - } - - sib->color = parent->color; - parent->color = IBV_BLACK; - if (sib->left) - sib->left->color = IBV_BLACK; - __mm_rotate_right(parent); - child = mm_root; - break; - } - } - } - - if (child) - child->color = IBV_BLACK; -} - -static struct ibv_mem_node *__mm_find_start(uintptr_t start, uintptr_t end) -{ - struct ibv_mem_node *node = mm_root; - - while (node) { - if (node->start <= start && node->end >= start) - break; - - if (node->start < start) - node = node->right; - else - node = node->left; - } - - return node; -} - -static struct ibv_mem_node *merge_ranges(struct ibv_mem_node *node, - struct ibv_mem_node *prev) -{ - prev->end = node->end; - prev->refcnt = node->refcnt; - __mm_remove(node); - - return prev; -} - -static struct ibv_mem_node *split_range(struct ibv_mem_node *node, - uintptr_t cut_line) -{ - struct ibv_mem_node *new_node = NULL; - - new_node = malloc(sizeof *new_node); - if (!new_node) - return NULL; - new_node->start = cut_line; - new_node->end = node->end; - new_node->refcnt = node->refcnt; - node->end = cut_line - 1; - __mm_add(new_node); - - return new_node; -} - -static struct ibv_mem_node *get_start_node(uintptr_t start, uintptr_t end, - int inc) -{ - struct ibv_mem_node *node, *tmp = NULL; - - node = __mm_find_start(start, end); - if (node->start < start) - node = split_range(node, start); - else { - tmp = __mm_prev(node); - if (tmp && tmp->refcnt == node->refcnt + inc) - node = merge_ranges(node, tmp); - } - return node; -} - -/* - * This function is called if madvise() fails to undo merging/splitting - * operations performed on the node. - */ -static struct ibv_mem_node *undo_node(struct ibv_mem_node *node, - uintptr_t start, int inc) -{ - struct ibv_mem_node *tmp = NULL; - - /* - * This condition can be true only if we merged this - * node with the previous one, so we need to split them. - */ - if (start > node->start) { - tmp = split_range(node, start); - if (tmp) { - node->refcnt += inc; - node = tmp; - } else - return NULL; - } - - tmp = __mm_prev(node); - if (tmp && tmp->refcnt == node->refcnt) - node = merge_ranges(node, tmp); - - tmp = __mm_next(node); - if (tmp && tmp->refcnt == node->refcnt) - node = merge_ranges(tmp, node); - - return node; -} - -static int ibv_madvise_range(void *base, size_t size, int advice) -{ - uintptr_t start, end; - struct ibv_mem_node *node, *tmp; - int inc; - int rolling_back = 0; - int ret = 0; - unsigned long range_page_size; - - if (!size) - return 0; - - if (huge_page_enabled) - range_page_size = get_page_size(base); - else - range_page_size = page_size; - - start = (uintptr_t) base & ~(range_page_size - 1); - end = ((uintptr_t) (base + size + range_page_size - 1) & - ~(range_page_size - 1)) - 1; - - pthread_mutex_lock(&mm_mutex); -again: - inc = advice == MADV_DONTFORK ? 1 : -1; - - node = get_start_node(start, end, inc); - if (!node) { - ret = -1; - goto out; - } - - while (node && node->start <= end) { - if (node->end > end) { - if (!split_range(node, end + 1)) { - ret = -1; - goto out; - } - } - - if ((inc == -1 && node->refcnt == 1) || - (inc == 1 && node->refcnt == 0)) { - /* - * If this is the first time through the loop, - * and we merged this node with the previous - * one, then we only want to do the madvise() - * on start ... node->end (rather than - * starting at node->start). - * - * Otherwise we end up doing madvise() on - * bigger region than we're being asked to, - * and that may lead to a spurious failure. - */ - if (start > node->start) - ret = madvise((void *) start, node->end - start + 1, - advice); - else - ret = madvise((void *) node->start, - node->end - node->start + 1, - advice); - if (ret) { - node = undo_node(node, start, inc); - - if (rolling_back || !node) - goto out; - - /* madvise failed, roll back previous changes */ - rolling_back = 1; - advice = advice == MADV_DONTFORK ? - MADV_DOFORK : MADV_DONTFORK; - tmp = __mm_prev(node); - if (!tmp || start > tmp->end) - goto out; - end = tmp->end; - goto again; - } - } - - node->refcnt += inc; - node = __mm_next(node); - } - - if (node) { - tmp = __mm_prev(node); - if (tmp && node->refcnt == tmp->refcnt) - node = merge_ranges(node, tmp); - } - -out: - if (rolling_back) - ret = -1; - - pthread_mutex_unlock(&mm_mutex); - - return ret; -} - -int ibv_dontfork_range(void *base, size_t size) -{ - if (mm_root) - return ibv_madvise_range(base, size, MADV_DONTFORK); - else { - too_late = 1; - return 0; - } -} - -int ibv_dofork_range(void *base, size_t size) -{ - if (mm_root) - return ibv_madvise_range(base, size, MADV_DOFORK); - else { - too_late = 1; - return 0; - } -} diff --git a/src/neigh.c b/src/neigh.c deleted file mode 100644 index 799b810..0000000 --- a/src/neigh.c +++ /dev/null @@ -1,853 +0,0 @@ - -#include "config.h" -#include <net/if_packet.h> -#include <linux/netlink.h> -#include <linux/rtnetlink.h> -#include <stdio.h> -#include <stdlib.h> -#include <stdbool.h> -#include <netlink/route/rtnl.h> -#include <netlink/route/link.h> -#include <netlink/route/route.h> -#include <netlink/route/neighbour.h> - -#include <sys/types.h> -#include <sys/socket.h> -#include <sys/timerfd.h> -#include <netinet/in.h> -#include <errno.h> -#include <unistd.h> -#include <ifaddrs.h> -#include <netdb.h> -#ifndef _LINUX_IF_H -#include <net/if.h> -#else -/*Workaround when there's a collision between the includes */ -extern unsigned int if_nametoindex(__const char *__ifname) __THROW; -#endif - -/* for PFX */ -#include "ibverbs.h" - -#include "neigh.h" - -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -/* Workaround - declaration missing */ -extern int rtnl_link_vlan_get_id(struct rtnl_link *); - -#ifndef HAVE_LIBNL1 -#include <netlink/route/link/vlan.h> -#endif - -static pthread_once_t device_neigh_alloc = PTHREAD_ONCE_INIT; -static struct nl_sock *zero_socket; - -union sktaddr { - struct sockaddr s; - struct sockaddr_in s4; - struct sockaddr_in6 s6; -}; - -struct skt { - union sktaddr sktaddr; - socklen_t len; -}; - -static int set_link_port(union sktaddr *s, int port, int oif) -{ - switch (s->s.sa_family) { - case AF_INET: - s->s4.sin_port = port; - break; - case AF_INET6: - s->s6.sin6_port = port; - s->s6.sin6_scope_id = oif; - break; - default: - return -EINVAL; - } - - return 0; -} - -static bool cmp_address(const struct sockaddr *s1, - const struct sockaddr *s2) -{ - if (s1->sa_family != s2->sa_family) - return false; - - switch (s1->sa_family) { - case AF_INET: - return ((struct sockaddr_in *)s1)->sin_addr.s_addr == - ((struct sockaddr_in *)s2)->sin_addr.s_addr; - case AF_INET6: - return !memcmp( - ((struct sockaddr_in6 *)s1)->sin6_addr.s6_addr, - ((struct sockaddr_in6 *)s2)->sin6_addr.s6_addr, - sizeof(((struct sockaddr_in6 *)s1)->sin6_addr.s6_addr)); - default: - return false; - } -} - -static int get_ifindex(const struct sockaddr *s) -{ - struct ifaddrs *ifaddr, *ifa; - int name2index = -ENODEV; - - if (-1 == getifaddrs(&ifaddr)) - return errno; - - for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == NULL) - continue; - - if (cmp_address(ifa->ifa_addr, s)) { - name2index = if_nametoindex(ifa->ifa_name); - break; - } - } - - freeifaddrs(ifaddr); - - return name2index; -} - -static struct nl_addr *get_neigh_mac(struct get_neigh_handler *neigh_handler) -{ - struct rtnl_neigh *neigh; - struct nl_addr *ll_addr = NULL; - - /* future optimization - if link local address - parse address and - * return mac now instead of doing so after the routing CB. This - * is of course referred to GIDs */ - neigh = rtnl_neigh_get(neigh_handler->neigh_cache, - neigh_handler->oif, - neigh_handler->dst); - if (neigh == NULL) - return NULL; - - ll_addr = rtnl_neigh_get_lladdr(neigh); - if (NULL != ll_addr) - ll_addr = nl_addr_clone(ll_addr); - - rtnl_neigh_put(neigh); - return ll_addr; -} - -static void get_neigh_cb_event(struct nl_object *obj, void *arg) -{ - struct get_neigh_handler *neigh_handler = - (struct get_neigh_handler *)arg; - /* assumed serilized callback (no parallel execution of function) */ - if (nl_object_match_filter( - obj, - (struct nl_object *)neigh_handler->filter_neigh)) { - struct rtnl_neigh *neigh = (struct rtnl_neigh *)obj; - /* check that we didn't set it already */ - if (neigh_handler->found_ll_addr == NULL) { - if (rtnl_neigh_get_lladdr(neigh) == NULL) - return; - - neigh_handler->found_ll_addr = - nl_addr_clone(rtnl_neigh_get_lladdr(neigh)); - } - } -} - -static int get_neigh_cb(struct nl_msg *msg, void *arg) -{ - struct get_neigh_handler *neigh_handler = - (struct get_neigh_handler *)arg; - - if (nl_msg_parse(msg, &get_neigh_cb_event, neigh_handler) < 0) - errno = ENOMSG; - - return NL_OK; -} - -static void set_neigh_filter(struct get_neigh_handler *neigh_handler, - struct rtnl_neigh *filter) { - neigh_handler->filter_neigh = filter; -} - -static struct rtnl_neigh *create_filter_neigh_for_dst(struct nl_addr *dst_addr, - int oif) -{ - struct rtnl_neigh *filter_neigh; - - filter_neigh = rtnl_neigh_alloc(); - if (filter_neigh == NULL) - return NULL; - - rtnl_neigh_set_ifindex(filter_neigh, oif); - rtnl_neigh_set_dst(filter_neigh, dst_addr); - - return filter_neigh; -} - -#define PORT_DISCARD htons(9) -#define SEND_PAYLOAD "H" - -static int create_socket(struct get_neigh_handler *neigh_handler, - struct skt *addr_dst, int *psock_fd) -{ - int err; - struct skt addr_src; - int sock_fd; - - memset(addr_dst, 0, sizeof(*addr_dst)); - memset(&addr_src, 0, sizeof(addr_src)); - addr_src.len = sizeof(addr_src.sktaddr); - - err = nl_addr_fill_sockaddr(neigh_handler->src, - &addr_src.sktaddr.s, - &addr_src.len); - if (err) { - errno = EADDRNOTAVAIL; - return err; - } - - addr_dst->len = sizeof(addr_dst->sktaddr); - err = nl_addr_fill_sockaddr(neigh_handler->dst, - &addr_dst->sktaddr.s, - &addr_dst->len); - if (err) { - errno = EADDRNOTAVAIL; - return err; - } - - err = set_link_port(&addr_dst->sktaddr, PORT_DISCARD, - neigh_handler->oif); - if (err) - return err; - - sock_fd = socket(addr_dst->sktaddr.s.sa_family, - SOCK_DGRAM | SOCK_CLOEXEC, 0); - if (sock_fd == -1) - return errno ? -errno : -1; - err = bind(sock_fd, &addr_src.sktaddr.s, addr_src.len); - if (err) { - int bind_err = -errno; - - close(sock_fd); - return bind_err ?: EADDRNOTAVAIL; - } - - *psock_fd = sock_fd; - - return 0; -} - -#define NUM_OF_RETRIES 10 -#define NUM_OF_TRIES ((NUM_OF_RETRIES) + 1) -#if NUM_OF_TRIES < 1 -#error "neigh: invalid value of NUM_OF_RETRIES" -#endif -static int create_timer(struct get_neigh_handler *neigh_handler) -{ - int user_timeout = neigh_handler->timeout/NUM_OF_TRIES; - struct timespec timeout = { - .tv_sec = user_timeout / 1000, - .tv_nsec = (user_timeout % 1000) * 1000000 - }; - struct itimerspec timer_time = {.it_value = timeout}; - int timer_fd; - - timer_fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC); - if (timer_fd == -1) - return timer_fd; - - if (neigh_handler->timeout) { - if (NUM_OF_TRIES <= 1) - bzero(&timer_time.it_interval, - sizeof(timer_time.it_interval)); - else - timer_time.it_interval = timeout; - if (timerfd_settime(timer_fd, 0, &timer_time, NULL)) { - close(timer_fd); - return -1; - } - } - - return timer_fd; -} - -#define UDP_SOCKET_MAX_SENDTO 100000ULL -static int try_send_to(int sock_fd, void *buff, size_t buf_size, - struct skt *addr_dst) -{ - uint64_t max_count = UDP_SOCKET_MAX_SENDTO; - int err; - - do { - err = sendto(sock_fd, buff, buf_size, 0, - &addr_dst->sktaddr.s, - addr_dst->len); - if (err > 0) - err = 0; - } while (-1 == err && EADDRNOTAVAIL == errno && --max_count); - - return err; -} - -static struct nl_addr *process_get_neigh_mac( - struct get_neigh_handler *neigh_handler) -{ - int err; - struct nl_addr *ll_addr = get_neigh_mac(neigh_handler); - struct rtnl_neigh *neigh_filter; - fd_set fdset; - int sock_fd; - int fd; - int nfds; - int timer_fd; - int ret; - struct skt addr_dst; - char buff[sizeof(SEND_PAYLOAD)] = SEND_PAYLOAD; - int retries = 0; - - if (NULL != ll_addr) - return ll_addr; - - err = nl_socket_add_membership(neigh_handler->sock, - RTNLGRP_NEIGH); - if (err < 0) - return NULL; - - neigh_filter = create_filter_neigh_for_dst(neigh_handler->dst, - neigh_handler->oif); - if (neigh_filter == NULL) - return NULL; - - set_neigh_filter(neigh_handler, neigh_filter); - - nl_socket_disable_seq_check(neigh_handler->sock); - nl_socket_modify_cb(neigh_handler->sock, NL_CB_VALID, NL_CB_CUSTOM, - &get_neigh_cb, neigh_handler); - - fd = nl_socket_get_fd(neigh_handler->sock); - - err = create_socket(neigh_handler, &addr_dst, &sock_fd); - - if (err) - return NULL; - - err = try_send_to(sock_fd, buff, sizeof(buff), &addr_dst); - if (err) - goto close_socket; - - timer_fd = create_timer(neigh_handler); - if (timer_fd < 0) - goto close_socket; - - nfds = MAX(fd, timer_fd) + 1; - - while (1) { - FD_ZERO(&fdset); - FD_SET(fd, &fdset); - FD_SET(timer_fd, &fdset); - - /* wait for an incoming message on the netlink socket */ - ret = select(nfds, &fdset, NULL, NULL, NULL); - if (ret == -1) { - goto select_err; - } else if (ret) { - if (FD_ISSET(fd, &fdset)) { - nl_recvmsgs_default(neigh_handler->sock); - if (neigh_handler->found_ll_addr) - break; - } else { - nl_cache_refill(neigh_handler->sock, - neigh_handler->neigh_cache); - ll_addr = get_neigh_mac(neigh_handler); - if (NULL != ll_addr) { - break; - } else if (FD_ISSET(timer_fd, &fdset) && - retries < NUM_OF_RETRIES) { - try_send_to(sock_fd, buff, sizeof(buff), - &addr_dst); - } - } - - if (FD_ISSET(timer_fd, &fdset)) { - uint64_t read_val; - - (void)read(timer_fd, &read_val, - sizeof(read_val)); - if (++retries >= NUM_OF_TRIES) { - if (!errno) - errno = EDESTADDRREQ; - break; - } - } - } - } -select_err: - close(timer_fd); -close_socket: - close(sock_fd); - return ll_addr ? ll_addr : neigh_handler->found_ll_addr; -} - -static int get_mcast_mac_ipv4(struct nl_addr *dst, struct nl_addr **ll_addr) -{ - uint8_t mac_addr[6] = {0x01, 0x00, 0x5E}; - uint32_t addr = ntohl(*(uint32_t *)nl_addr_get_binary_addr(dst)); - - mac_addr[5] = addr & 0xFF; - addr >>= 8; - mac_addr[4] = addr & 0xFF; - addr >>= 8; - mac_addr[3] = addr & 0x7F; - - *ll_addr = nl_addr_build(AF_LLC, mac_addr, sizeof(mac_addr)); - - return *ll_addr == NULL ? -EINVAL : 0; -} - -static int get_mcast_mac_ipv6(struct nl_addr *dst, struct nl_addr **ll_addr) -{ - uint8_t mac_addr[6] = {0x33, 0x33}; - - memcpy(mac_addr + 2, (uint8_t *)nl_addr_get_binary_addr(dst) + 12, 4); - - *ll_addr = nl_addr_build(AF_LLC, mac_addr, sizeof(mac_addr)); - - return *ll_addr == NULL ? -EINVAL : 0; -} - -static int get_link_local_mac_ipv6(struct nl_addr *dst, - struct nl_addr **ll_addr) -{ - uint8_t mac_addr[6]; - - memcpy(mac_addr + 3, (uint8_t *)nl_addr_get_binary_addr(dst) + 13, 3); - memcpy(mac_addr, (uint8_t *)nl_addr_get_binary_addr(dst) + 8, 3); - mac_addr[0] ^= 2; - - *ll_addr = nl_addr_build(AF_LLC, mac_addr, sizeof(mac_addr)); - return *ll_addr == NULL ? -EINVAL : 0; -} - -static const struct encoded_l3_addr { - short family; - uint8_t prefix_bits; - const uint8_t data[16]; - int (*getter)(struct nl_addr *dst, struct nl_addr **ll_addr); -} encoded_prefixes[] = { - {.family = AF_INET, - .prefix_bits = 4, - .data = {0xe0}, - .getter = &get_mcast_mac_ipv4}, - {.family = AF_INET6, - .prefix_bits = 8, - .data = {0xff}, - .getter = &get_mcast_mac_ipv6}, - {.family = AF_INET6, - .prefix_bits = 64, - .data = {0xfe, 0x80}, - .getter = get_link_local_mac_ipv6}, -}; - -static int nl_addr_cmp_prefix_msb(void *addr1, int len1, void *addr2, int len2) -{ - int len = MIN(len1, len2); - int bytes = len / 8; - int d = memcmp(addr1, addr2, bytes); - - if (d == 0) { - int mask = ((1UL << (len % 8)) - 1UL) << (8 - len); - - d = (((uint8_t *)addr1)[bytes] & mask) - - (((uint8_t *)addr2)[bytes] & mask); - } - - return d; -} - -static int handle_encoded_mac(struct nl_addr *dst, struct nl_addr **ll_addr) -{ - uint32_t family = nl_addr_get_family(dst); - struct nl_addr *prefix = NULL; - int i; - int ret = 1; - - for (i = 0; - i < sizeof(encoded_prefixes)/sizeof(encoded_prefixes[0]) && - ret; prefix = NULL, i++) { - if (encoded_prefixes[i].family != family) - continue; - - prefix = nl_addr_build( - family, - (void *)encoded_prefixes[i].data, - MIN(encoded_prefixes[i].prefix_bits/8 + - !!(encoded_prefixes[i].prefix_bits % 8), - sizeof(encoded_prefixes[i].data))); - - if (prefix == NULL) - return -ENOMEM; - nl_addr_set_prefixlen(prefix, - encoded_prefixes[i].prefix_bits); - - if (nl_addr_cmp_prefix_msb(nl_addr_get_binary_addr(dst), - nl_addr_get_prefixlen(dst), - nl_addr_get_binary_addr(prefix), - nl_addr_get_prefixlen(prefix))) - continue; - - ret = encoded_prefixes[i].getter(dst, ll_addr); - nl_addr_put(prefix); - } - - return ret; -} - -static void get_route_cb_parser(struct nl_object *obj, void *arg) -{ - struct get_neigh_handler *neigh_handler = - (struct get_neigh_handler *)arg; - - struct rtnl_route *route = (struct rtnl_route *)obj; - struct nl_addr *gateway = NULL; - struct nl_addr *src = rtnl_route_get_pref_src(route); - int oif; - int type = rtnl_route_get_type(route); - struct rtnl_link *link; - - struct rtnl_nexthop *nh = rtnl_route_nexthop_n(route, 0); - - if (nh != NULL) - gateway = rtnl_route_nh_get_gateway(nh); - oif = rtnl_route_nh_get_ifindex(nh); - - if (gateway) { - nl_addr_put(neigh_handler->dst); - neigh_handler->dst = nl_addr_clone(gateway); - } - - if (RTN_BLACKHOLE == type || - RTN_UNREACHABLE == type || - RTN_PROHIBIT == type || - RTN_THROW == type) { - errno = ENETUNREACH; - goto err; - } - - if (!neigh_handler->src && src) - neigh_handler->src = nl_addr_clone(src); - - if (neigh_handler->oif < 0 && oif > 0) - neigh_handler->oif = oif; - - /* Link Local */ - if (RTN_LOCAL == type) { - struct nl_addr *lladdr; - - link = rtnl_link_get(neigh_handler->link_cache, - neigh_handler->oif); - - if (link == NULL) - goto err; - - lladdr = rtnl_link_get_addr(link); - - if (lladdr == NULL) - goto err_link; - - neigh_handler->found_ll_addr = nl_addr_clone(lladdr); - rtnl_link_put(link); - } else { - handle_encoded_mac( - neigh_handler->dst, - &neigh_handler->found_ll_addr); - } - - return; - -err_link: - rtnl_link_put(link); -err: - if (neigh_handler->src) { - nl_addr_put(neigh_handler->src); - neigh_handler->src = NULL; - } -} - -static int get_route_cb(struct nl_msg *msg, void *arg) -{ - struct get_neigh_handler *neigh_handler = - (struct get_neigh_handler *)arg; - int err; - - err = nl_msg_parse(msg, &get_route_cb_parser, neigh_handler); - if (err < 0) { - errno = ENOMSG; - return err; - } - - if (!neigh_handler->dst || !neigh_handler->src || - neigh_handler->oif <= 0) { - errno = EINVAL; - return -1; - } - - if (NULL != neigh_handler->found_ll_addr) - goto found; - - neigh_handler->found_ll_addr = - process_get_neigh_mac(neigh_handler); - -found: - return neigh_handler->found_ll_addr ? 0 : -1; -} - -int neigh_get_oif_from_src(struct get_neigh_handler *neigh_handler) -{ - int oif = -ENODEV; - struct addrinfo *src_info; - int err; - - err = nl_addr_info(neigh_handler->src, &src_info); - if (err) { - if (!errno) - errno = ENXIO; - return oif; - } - - oif = get_ifindex(src_info->ai_addr); - if (oif <= 0) - goto free; - -free: - freeaddrinfo(src_info); - return oif; -} - -static void destroy_zero_based_socket(void) -{ - if (zero_socket != NULL) - nl_socket_free(zero_socket); -} - -static void alloc_zero_based_socket(void) -{ - zero_socket = nl_socket_alloc(); - atexit(&destroy_zero_based_socket); -} - -int neigh_init_resources(struct get_neigh_handler *neigh_handler, int timeout) -{ - int err; - - pthread_once(&device_neigh_alloc, &alloc_zero_based_socket); - neigh_handler->sock = nl_socket_alloc(); - if (neigh_handler->sock == NULL) { - errno = ENOMEM; - return -1; - } - - err = nl_connect(neigh_handler->sock, NETLINK_ROUTE); - if (err < 0) - goto free_socket; - - err = rtnl_link_alloc_cache(neigh_handler->sock, AF_UNSPEC, - &neigh_handler->link_cache); - if (err) { - err = -1; - errno = ENOMEM; - goto close_connection; - } - - nl_cache_mngt_provide(neigh_handler->link_cache); - - err = rtnl_route_alloc_cache(neigh_handler->sock, AF_UNSPEC, 0, - &neigh_handler->route_cache); - if (err) { - err = -1; - errno = ENOMEM; - goto free_link_cache; - } - - nl_cache_mngt_provide(neigh_handler->route_cache); - - err = rtnl_neigh_alloc_cache(neigh_handler->sock, - &neigh_handler->neigh_cache); - if (err) { - err = -ENOMEM; - goto free_route_cache; - } - - nl_cache_mngt_provide(neigh_handler->neigh_cache); - - /* init structure */ - neigh_handler->timeout = timeout; - neigh_handler->oif = -1; - neigh_handler->filter_neigh = NULL; - neigh_handler->found_ll_addr = NULL; - neigh_handler->dst = NULL; - neigh_handler->src = NULL; - neigh_handler->vid = -1; - - return 0; - -free_route_cache: - nl_cache_mngt_unprovide(neigh_handler->route_cache); - nl_cache_free(neigh_handler->route_cache); - neigh_handler->route_cache = NULL; -free_link_cache: - nl_cache_mngt_unprovide(neigh_handler->link_cache); - nl_cache_free(neigh_handler->link_cache); - neigh_handler->link_cache = NULL; -close_connection: - nl_close(neigh_handler->sock); -free_socket: - nl_socket_free(neigh_handler->sock); - neigh_handler->sock = NULL; - return err; -} - -uint16_t neigh_get_vlan_id_from_dev(struct get_neigh_handler *neigh_handler) -{ - struct rtnl_link *link; - int vid = 0xffff; - - link = rtnl_link_get(neigh_handler->link_cache, neigh_handler->oif); - if (link == NULL) { - errno = EINVAL; - return vid; - } - - if (rtnl_link_is_vlan(link)) - vid = rtnl_link_vlan_get_id(link); - rtnl_link_put(link); - return vid >= 0 && vid <= 0xfff ? vid : 0xffff; -} - -void neigh_set_vlan_id(struct get_neigh_handler *neigh_handler, uint16_t vid) -{ - if (vid >= 0 && vid <= 0xfff) - neigh_handler->vid = vid; -} - -int neigh_set_dst(struct get_neigh_handler *neigh_handler, - int family, void *buf, size_t size) -{ - neigh_handler->dst = nl_addr_build(family, buf, size); - return neigh_handler->dst == NULL; -} - -int neigh_set_src(struct get_neigh_handler *neigh_handler, - int family, void *buf, size_t size) -{ - neigh_handler->src = nl_addr_build(family, buf, size); - return neigh_handler->src == NULL; -} - -void neigh_set_oif(struct get_neigh_handler *neigh_handler, int oif) -{ - neigh_handler->oif = oif; -} - -int neigh_get_ll(struct get_neigh_handler *neigh_handler, void *addr_buff, - int addr_size) { - int neigh_len; - - if (neigh_handler->found_ll_addr == NULL) - return -EINVAL; - - neigh_len = nl_addr_get_len(neigh_handler->found_ll_addr); - - if (neigh_len > addr_size) - return -EINVAL; - - memcpy(addr_buff, nl_addr_get_binary_addr(neigh_handler->found_ll_addr), - neigh_len); - - return neigh_len; -} - -void neigh_free_resources(struct get_neigh_handler *neigh_handler) -{ - /* Should be released first because it's holding a reference to dst */ - if (neigh_handler->filter_neigh != NULL) { - rtnl_neigh_put(neigh_handler->filter_neigh); - neigh_handler->filter_neigh = NULL; - } - - if (neigh_handler->src != NULL) { - nl_addr_put(neigh_handler->src); - neigh_handler->src = NULL; - } - - if (neigh_handler->dst != NULL) { - nl_addr_put(neigh_handler->dst); - neigh_handler->dst = NULL; - } - - if (neigh_handler->found_ll_addr != NULL) { - nl_addr_put(neigh_handler->found_ll_addr); - neigh_handler->found_ll_addr = NULL; - } - - if (neigh_handler->neigh_cache != NULL) { - nl_cache_mngt_unprovide(neigh_handler->neigh_cache); - nl_cache_free(neigh_handler->neigh_cache); - neigh_handler->neigh_cache = NULL; - } - - if (neigh_handler->route_cache != NULL) { - nl_cache_mngt_unprovide(neigh_handler->route_cache); - nl_cache_free(neigh_handler->route_cache); - neigh_handler->route_cache = NULL; - } - - if (neigh_handler->link_cache != NULL) { - nl_cache_mngt_unprovide(neigh_handler->link_cache); - nl_cache_free(neigh_handler->link_cache); - neigh_handler->link_cache = NULL; - } - - if (neigh_handler->sock != NULL) { - nl_close(neigh_handler->sock); - nl_socket_free(neigh_handler->sock); - neigh_handler->sock = NULL; - } -} - -int process_get_neigh(struct get_neigh_handler *neigh_handler) -{ - struct nl_msg *m; - struct rtmsg rmsg = { - .rtm_family = nl_addr_get_family(neigh_handler->dst), - .rtm_dst_len = nl_addr_get_prefixlen(neigh_handler->dst), - }; - int err; - - m = nlmsg_alloc_simple(RTM_GETROUTE, 0); - - if (m == NULL) - return -ENOMEM; - - nlmsg_append(m, &rmsg, sizeof(rmsg), NLMSG_ALIGNTO); - - nla_put_addr(m, RTA_DST, neigh_handler->dst); - - if (neigh_handler->oif > 0) - nla_put_u32(m, RTA_OIF, neigh_handler->oif); - - err = nl_send_auto_complete(neigh_handler->sock, m); - nlmsg_free(m); - if (err < 0) - return err; - - nl_socket_modify_cb(neigh_handler->sock, NL_CB_VALID, - NL_CB_CUSTOM, &get_route_cb, neigh_handler); - - err = nl_recvmsgs_default(neigh_handler->sock); - - return err; -} diff --git a/src/neigh.h b/src/neigh.h deleted file mode 100644 index 61dfd85..0000000 --- a/src/neigh.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef _NEIGH_H_ -#define _NEIGH_H_ - -#include <stddef.h> -#include <stdint.h> -#include "config.h" -#ifdef HAVE_LIBNL1 -#include <netlink/object.h> -#include "nl1_compat.h" -#else -#include <netlink/object-api.h> -#endif - -struct get_neigh_handler { -#ifdef HAVE_LIBNL1 - struct nl_handle *sock; -#else - struct nl_sock *sock; -#endif - struct nl_cache *link_cache; - struct nl_cache *neigh_cache; - struct nl_cache *route_cache; - int32_t oif; - int vid; - struct rtnl_neigh *filter_neigh; - struct nl_addr *found_ll_addr; - struct nl_addr *dst; - struct nl_addr *src; - uint64_t timeout; -}; - -int process_get_neigh(struct get_neigh_handler *neigh_handler); -void neigh_free_resources(struct get_neigh_handler *neigh_handler); -void neigh_set_vlan_id(struct get_neigh_handler *neigh_handler, uint16_t vid); -uint16_t neigh_get_vlan_id_from_dev(struct get_neigh_handler *neigh_handler); -int neigh_init_resources(struct get_neigh_handler *neigh_handler, int timeout); - -int neigh_set_src(struct get_neigh_handler *neigh_handler, - int family, void *buf, size_t size); -void neigh_set_oif(struct get_neigh_handler *neigh_handler, int oif); -int neigh_set_dst(struct get_neigh_handler *neigh_handler, - int family, void *buf, size_t size); -int neigh_get_oif_from_src(struct get_neigh_handler *neigh_handler); -int neigh_get_ll(struct get_neigh_handler *neigh_handler, void *addr_buf, - int addr_size); - -#endif diff --git a/src/nl1_compat.h b/src/nl1_compat.h deleted file mode 100644 index 809259d..0000000 --- a/src/nl1_compat.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef _NL1_COMPAT_H_ -#define _NL1_COMPAT_H_ - -#include <netlink/route/rtnl.h> -#include <netlink/route/link.h> -#include <netlink/route/route.h> -#include <netlink/route/neighbour.h> - -/* Workaround - declaration missing */ -extern int rtnl_link_vlan_get_id(struct rtnl_link *); - -#define nl_geterror(x) nl_geterror() -#define nl_sock nl_handle - -static inline void nl_socket_disable_seq_check(struct nl_sock *sock) -{ - nl_disable_sequence_check(sock); -} - -struct rtnl_nexthop {}; - -static inline struct rtnl_nexthop *rtnl_route_nexthop_n( - struct rtnl_route *r, int n) -{ - return (struct rtnl_nexthop *)r; -} - -static inline struct nl_addr *rtnl_route_nh_get_gateway(struct rtnl_nexthop *nh) -{ - return rtnl_route_get_gateway((struct rtnl_route *)nh); -} - -static inline int rtnl_route_nh_get_ifindex(struct rtnl_nexthop *nh) -{ - return rtnl_route_get_oif((struct rtnl_route *)nh); -} - -#define nl_addr_info(addr, result) ( \ - *(result) = nl_addr_info(addr), \ - (*(result) == NULL) ? nl_get_errno() : 0 \ -) - -static inline void nl_socket_free(struct nl_sock *sock) -{ - nl_close(sock); -} - -static inline struct nl_sock *nl_socket_alloc(void) -{ - return nl_handle_alloc(); -} - -#define rtnl_link_alloc_cache(sock, family, result) ( \ - *result = rtnl_link_alloc_cache(sock), \ - (*result == NULL) ? nl_get_errno() : 0 \ -) - -#define rtnl_route_alloc_cache(sock, family, flags, result) ( \ - *result = rtnl_route_alloc_cache(sock), \ - (*result == NULL) ? nl_get_errno() : 0 \ -) - -#define rtnl_neigh_alloc_cache(sock, result) ( \ - *result = rtnl_neigh_alloc_cache(sock), \ - (*result == NULL) ? nl_get_errno() : 0 \ -) - -static inline int rtnl_link_is_vlan(struct rtnl_link *link) -{ - return rtnl_link_vlan_get_id(link) <= 0; -} - -#endif diff --git a/src/sysfs.c b/src/sysfs.c deleted file mode 100644 index 0b6ad1e..0000000 --- a/src/sysfs.c +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <string.h> - -#include "ibverbs.h" - -static char *sysfs_path; - -const char *ibv_get_sysfs_path(void) -{ - char *env = NULL; - - if (sysfs_path) - return sysfs_path; - - /* - * Only follow use path passed in through the calling user's - * environment if we're not running SUID. - */ - if (getuid() == geteuid()) - env = getenv("SYSFS_PATH"); - - if (env) { - int len; - - sysfs_path = strndup(env, IBV_SYSFS_PATH_MAX); - len = strlen(sysfs_path); - while (len > 0 && sysfs_path[len - 1] == '/') { - --len; - sysfs_path[len] = '\0'; - } - } else - sysfs_path = "/sys"; - - return sysfs_path; -} - -int ibv_read_sysfs_file(const char *dir, const char *file, - char *buf, size_t size) -{ - char *path; - int fd; - int len; - - if (asprintf(&path, "%s/%s", dir, file) < 0) - return -1; - - fd = open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) { - free(path); - return -1; - } - - len = read(fd, buf, size); - - close(fd); - free(path); - - if (len > 0) { - if (buf[len - 1] == '\n') - buf[--len] = '\0'; - else if (len < size) - buf[len] = '\0'; - else - /* We would have to truncate the contents to NULL - * terminate, so we are going to fail no matter - * what we do, either right now or later when - * we pass around an unterminated string. Fail now. - */ - return -1; - } - - return len; -} diff --git a/src/verbs.c b/src/verbs.c deleted file mode 100644 index fc8b9f6..0000000 --- a/src/verbs.c +++ /dev/null @@ -1,987 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#if HAVE_CONFIG_H -# include <config.h> -#endif /* HAVE_CONFIG_H */ - -#include <stdio.h> -#include <netinet/in.h> -#include <unistd.h> -#include <stdlib.h> -#include <errno.h> -#include <string.h> -#include <linux/ip.h> -#include <dirent.h> - -#include "ibverbs.h" -#ifndef NRESOLVE_NEIGH -#include <net/if.h> -#include <net/if_arp.h> -#include "neigh.h" -#endif - -int ibv_rate_to_mult(enum ibv_rate rate) -{ - switch (rate) { - case IBV_RATE_2_5_GBPS: return 1; - case IBV_RATE_5_GBPS: return 2; - case IBV_RATE_10_GBPS: return 4; - case IBV_RATE_20_GBPS: return 8; - case IBV_RATE_30_GBPS: return 12; - case IBV_RATE_40_GBPS: return 16; - case IBV_RATE_60_GBPS: return 24; - case IBV_RATE_80_GBPS: return 32; - case IBV_RATE_120_GBPS: return 48; - default: return -1; - } -} - -enum ibv_rate mult_to_ibv_rate(int mult) -{ - switch (mult) { - case 1: return IBV_RATE_2_5_GBPS; - case 2: return IBV_RATE_5_GBPS; - case 4: return IBV_RATE_10_GBPS; - case 8: return IBV_RATE_20_GBPS; - case 12: return IBV_RATE_30_GBPS; - case 16: return IBV_RATE_40_GBPS; - case 24: return IBV_RATE_60_GBPS; - case 32: return IBV_RATE_80_GBPS; - case 48: return IBV_RATE_120_GBPS; - default: return IBV_RATE_MAX; - } -} - -int ibv_rate_to_mbps(enum ibv_rate rate) -{ - switch (rate) { - case IBV_RATE_2_5_GBPS: return 2500; - case IBV_RATE_5_GBPS: return 5000; - case IBV_RATE_10_GBPS: return 10000; - case IBV_RATE_20_GBPS: return 20000; - case IBV_RATE_30_GBPS: return 30000; - case IBV_RATE_40_GBPS: return 40000; - case IBV_RATE_60_GBPS: return 60000; - case IBV_RATE_80_GBPS: return 80000; - case IBV_RATE_120_GBPS: return 120000; - case IBV_RATE_14_GBPS: return 14062; - case IBV_RATE_56_GBPS: return 56250; - case IBV_RATE_112_GBPS: return 112500; - case IBV_RATE_168_GBPS: return 168750; - case IBV_RATE_25_GBPS: return 25781; - case IBV_RATE_100_GBPS: return 103125; - case IBV_RATE_200_GBPS: return 206250; - case IBV_RATE_300_GBPS: return 309375; - default: return -1; - } -} - -enum ibv_rate mbps_to_ibv_rate(int mbps) -{ - switch (mbps) { - case 2500: return IBV_RATE_2_5_GBPS; - case 5000: return IBV_RATE_5_GBPS; - case 10000: return IBV_RATE_10_GBPS; - case 20000: return IBV_RATE_20_GBPS; - case 30000: return IBV_RATE_30_GBPS; - case 40000: return IBV_RATE_40_GBPS; - case 60000: return IBV_RATE_60_GBPS; - case 80000: return IBV_RATE_80_GBPS; - case 120000: return IBV_RATE_120_GBPS; - case 14062: return IBV_RATE_14_GBPS; - case 56250: return IBV_RATE_56_GBPS; - case 112500: return IBV_RATE_112_GBPS; - case 168750: return IBV_RATE_168_GBPS; - case 25781: return IBV_RATE_25_GBPS; - case 103125: return IBV_RATE_100_GBPS; - case 206250: return IBV_RATE_200_GBPS; - case 309375: return IBV_RATE_300_GBPS; - default: return IBV_RATE_MAX; - } -} - -int __ibv_query_device(struct ibv_context *context, - struct ibv_device_attr *device_attr) -{ - return context->ops.query_device(context, device_attr); -} -default_symver(__ibv_query_device, ibv_query_device); - -int __ibv_query_port(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr) -{ - return context->ops.query_port(context, port_num, port_attr); -} -default_symver(__ibv_query_port, ibv_query_port); - -int __ibv_query_gid(struct ibv_context *context, uint8_t port_num, - int index, union ibv_gid *gid) -{ - char name[24]; - char attr[41]; - uint16_t val; - int i; - - snprintf(name, sizeof name, "ports/%d/gids/%d", port_num, index); - - if (ibv_read_sysfs_file(context->device->ibdev_path, name, - attr, sizeof attr) < 0) - return -1; - - for (i = 0; i < 8; ++i) { - if (sscanf(attr + i * 5, "%hx", &val) != 1) - return -1; - gid->raw[i * 2 ] = val >> 8; - gid->raw[i * 2 + 1] = val & 0xff; - } - - return 0; -} -default_symver(__ibv_query_gid, ibv_query_gid); - -int __ibv_query_pkey(struct ibv_context *context, uint8_t port_num, - int index, uint16_t *pkey) -{ - char name[24]; - char attr[8]; - uint16_t val; - - snprintf(name, sizeof name, "ports/%d/pkeys/%d", port_num, index); - - if (ibv_read_sysfs_file(context->device->ibdev_path, name, - attr, sizeof attr) < 0) - return -1; - - if (sscanf(attr, "%hx", &val) != 1) - return -1; - - *pkey = htons(val); - return 0; -} -default_symver(__ibv_query_pkey, ibv_query_pkey); - -struct ibv_pd *__ibv_alloc_pd(struct ibv_context *context) -{ - struct ibv_pd *pd; - - pd = context->ops.alloc_pd(context); - if (pd) - pd->context = context; - - return pd; -} -default_symver(__ibv_alloc_pd, ibv_alloc_pd); - -int __ibv_dealloc_pd(struct ibv_pd *pd) -{ - return pd->context->ops.dealloc_pd(pd); -} -default_symver(__ibv_dealloc_pd, ibv_dealloc_pd); - -struct ibv_mr *__ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, int access) -{ - struct ibv_mr *mr; - - if (ibv_dontfork_range(addr, length)) - return NULL; - - mr = pd->context->ops.reg_mr(pd, addr, length, access); - if (mr) { - mr->context = pd->context; - mr->pd = pd; - mr->addr = addr; - mr->length = length; - } else - ibv_dofork_range(addr, length); - - return mr; -} -default_symver(__ibv_reg_mr, ibv_reg_mr); - -int __ibv_rereg_mr(struct ibv_mr *mr, int flags, - struct ibv_pd *pd, void *addr, - size_t length, int access) -{ - int dofork_onfail = 0; - int err; - void *old_addr; - size_t old_len; - - if (flags & ~IBV_REREG_MR_FLAGS_SUPPORTED) { - errno = EINVAL; - return IBV_REREG_MR_ERR_INPUT; - } - - if ((flags & IBV_REREG_MR_CHANGE_TRANSLATION) && - (!length || !addr)) { - errno = EINVAL; - return IBV_REREG_MR_ERR_INPUT; - } - - if (access && !(flags & IBV_REREG_MR_CHANGE_ACCESS)) { - errno = EINVAL; - return IBV_REREG_MR_ERR_INPUT; - } - - if (!mr->context->ops.rereg_mr) { - errno = ENOSYS; - return IBV_REREG_MR_ERR_INPUT; - } - - if (flags & IBV_REREG_MR_CHANGE_TRANSLATION) { - err = ibv_dontfork_range(addr, length); - if (err) - return IBV_REREG_MR_ERR_DONT_FORK_NEW; - dofork_onfail = 1; - } - - old_addr = mr->addr; - old_len = mr->length; - err = mr->context->ops.rereg_mr(mr, flags, pd, addr, length, access); - if (!err) { - if (flags & IBV_REREG_MR_CHANGE_PD) - mr->pd = pd; - if (flags & IBV_REREG_MR_CHANGE_TRANSLATION) { - mr->addr = addr; - mr->length = length; - err = ibv_dofork_range(old_addr, old_len); - if (err) - return IBV_REREG_MR_ERR_DO_FORK_OLD; - } - } else { - err = IBV_REREG_MR_ERR_CMD; - if (dofork_onfail) { - if (ibv_dofork_range(addr, length)) - err = IBV_REREG_MR_ERR_CMD_AND_DO_FORK_NEW; - } - } - - return err; -} -default_symver(__ibv_rereg_mr, ibv_rereg_mr); - -int __ibv_dereg_mr(struct ibv_mr *mr) -{ - int ret; - void *addr = mr->addr; - size_t length = mr->length; - - ret = mr->context->ops.dereg_mr(mr); - if (!ret) - ibv_dofork_range(addr, length); - - return ret; -} -default_symver(__ibv_dereg_mr, ibv_dereg_mr); - -static struct ibv_comp_channel *ibv_create_comp_channel_v2(struct ibv_context *context) -{ - struct ibv_abi_compat_v2 *t = context->abi_compat; - static int warned; - - if (!pthread_mutex_trylock(&t->in_use)) - return &t->channel; - - if (!warned) { - fprintf(stderr, PFX "Warning: kernel's ABI version %d limits capacity.\n" - " Only one completion channel can be created per context.\n", - abi_ver); - ++warned; - } - - return NULL; -} - -struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context) -{ - struct ibv_comp_channel *channel; - struct ibv_create_comp_channel cmd; - struct ibv_create_comp_channel_resp resp; - - if (abi_ver <= 2) - return ibv_create_comp_channel_v2(context); - - channel = malloc(sizeof *channel); - if (!channel) - return NULL; - - IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_COMP_CHANNEL, &resp, sizeof resp); - if (write(context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) { - free(channel); - return NULL; - } - - (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); - - channel->context = context; - channel->fd = resp.fd; - channel->refcnt = 0; - - return channel; -} - -static int ibv_destroy_comp_channel_v2(struct ibv_comp_channel *channel) -{ - struct ibv_abi_compat_v2 *t = (struct ibv_abi_compat_v2 *) channel; - pthread_mutex_unlock(&t->in_use); - return 0; -} - -int ibv_destroy_comp_channel(struct ibv_comp_channel *channel) -{ - struct ibv_context *context; - int ret; - - context = channel->context; - pthread_mutex_lock(&context->mutex); - - if (channel->refcnt) { - ret = EBUSY; - goto out; - } - - if (abi_ver <= 2) { - ret = ibv_destroy_comp_channel_v2(channel); - goto out; - } - - close(channel->fd); - free(channel); - ret = 0; - -out: - pthread_mutex_unlock(&context->mutex); - - return ret; -} - -struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context, - struct ibv_comp_channel *channel, int comp_vector) -{ - struct ibv_cq *cq; - - pthread_mutex_lock(&context->mutex); - - cq = context->ops.create_cq(context, cqe, channel, comp_vector); - - if (cq) { - cq->context = context; - cq->channel = channel; - if (channel) - ++channel->refcnt; - cq->cq_context = cq_context; - cq->comp_events_completed = 0; - cq->async_events_completed = 0; - pthread_mutex_init(&cq->mutex, NULL); - pthread_cond_init(&cq->cond, NULL); - } - - pthread_mutex_unlock(&context->mutex); - - return cq; -} -default_symver(__ibv_create_cq, ibv_create_cq); - -int __ibv_resize_cq(struct ibv_cq *cq, int cqe) -{ - if (!cq->context->ops.resize_cq) - return ENOSYS; - - return cq->context->ops.resize_cq(cq, cqe); -} -default_symver(__ibv_resize_cq, ibv_resize_cq); - -int __ibv_destroy_cq(struct ibv_cq *cq) -{ - struct ibv_comp_channel *channel = cq->channel; - int ret; - - if (channel) - pthread_mutex_lock(&channel->context->mutex); - - ret = cq->context->ops.destroy_cq(cq); - - if (channel) { - if (!ret) - --channel->refcnt; - pthread_mutex_unlock(&channel->context->mutex); - } - - return ret; -} -default_symver(__ibv_destroy_cq, ibv_destroy_cq); - -int __ibv_get_cq_event(struct ibv_comp_channel *channel, - struct ibv_cq **cq, void **cq_context) -{ - struct ibv_comp_event ev; - - if (read(channel->fd, &ev, sizeof ev) != sizeof ev) - return -1; - - *cq = (struct ibv_cq *) (uintptr_t) ev.cq_handle; - *cq_context = (*cq)->cq_context; - - if ((*cq)->context->ops.cq_event) - (*cq)->context->ops.cq_event(*cq); - - return 0; -} -default_symver(__ibv_get_cq_event, ibv_get_cq_event); - -void __ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) -{ - pthread_mutex_lock(&cq->mutex); - cq->comp_events_completed += nevents; - pthread_cond_signal(&cq->cond); - pthread_mutex_unlock(&cq->mutex); -} -default_symver(__ibv_ack_cq_events, ibv_ack_cq_events); - -struct ibv_srq *__ibv_create_srq(struct ibv_pd *pd, - struct ibv_srq_init_attr *srq_init_attr) -{ - struct ibv_srq *srq; - - if (!pd->context->ops.create_srq) - return NULL; - - srq = pd->context->ops.create_srq(pd, srq_init_attr); - if (srq) { - srq->context = pd->context; - srq->srq_context = srq_init_attr->srq_context; - srq->pd = pd; - srq->events_completed = 0; - pthread_mutex_init(&srq->mutex, NULL); - pthread_cond_init(&srq->cond, NULL); - } - - return srq; -} -default_symver(__ibv_create_srq, ibv_create_srq); - -int __ibv_modify_srq(struct ibv_srq *srq, - struct ibv_srq_attr *srq_attr, - int srq_attr_mask) -{ - return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask); -} -default_symver(__ibv_modify_srq, ibv_modify_srq); - -int __ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr) -{ - return srq->context->ops.query_srq(srq, srq_attr); -} -default_symver(__ibv_query_srq, ibv_query_srq); - -int __ibv_destroy_srq(struct ibv_srq *srq) -{ - return srq->context->ops.destroy_srq(srq); -} -default_symver(__ibv_destroy_srq, ibv_destroy_srq); - -struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, - struct ibv_qp_init_attr *qp_init_attr) -{ - struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr); - - if (qp) { - qp->context = pd->context; - qp->qp_context = qp_init_attr->qp_context; - qp->pd = pd; - qp->send_cq = qp_init_attr->send_cq; - qp->recv_cq = qp_init_attr->recv_cq; - qp->srq = qp_init_attr->srq; - qp->qp_type = qp_init_attr->qp_type; - qp->state = IBV_QPS_RESET; - qp->events_completed = 0; - pthread_mutex_init(&qp->mutex, NULL); - pthread_cond_init(&qp->cond, NULL); - } - - return qp; -} -default_symver(__ibv_create_qp, ibv_create_qp); - -int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask, - struct ibv_qp_init_attr *init_attr) -{ - int ret; - - ret = qp->context->ops.query_qp(qp, attr, attr_mask, init_attr); - if (ret) - return ret; - - if (attr_mask & IBV_QP_STATE) - qp->state = attr->qp_state; - - return 0; -} -default_symver(__ibv_query_qp, ibv_query_qp); - -int __ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - int attr_mask) -{ - int ret; - - ret = qp->context->ops.modify_qp(qp, attr, attr_mask); - if (ret) - return ret; - - if (attr_mask & IBV_QP_STATE) - qp->state = attr->qp_state; - - return 0; -} -default_symver(__ibv_modify_qp, ibv_modify_qp); - -int __ibv_destroy_qp(struct ibv_qp *qp) -{ - return qp->context->ops.destroy_qp(qp); -} -default_symver(__ibv_destroy_qp, ibv_destroy_qp); - -struct ibv_ah *__ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) -{ - struct ibv_ah *ah = pd->context->ops.create_ah(pd, attr); - - if (ah) { - ah->context = pd->context; - ah->pd = pd; - } - - return ah; -} -default_symver(__ibv_create_ah, ibv_create_ah); - -/* GID types as appear in sysfs, no change is expected as of ABI - * compatibility. - */ -#define V1_TYPE "IB/RoCE v1" -#define V2_TYPE "RoCE v2" -int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, - unsigned int index, enum ibv_gid_type *type) -{ - char name[32]; - char buff[11]; - - snprintf(name, sizeof(name), "ports/%d/gid_attrs/types/%d", port_num, - index); - - /* Reset errno so that we can rely on its value upon any error flow in - * ibv_read_sysfs_file. - */ - errno = 0; - if (ibv_read_sysfs_file(context->device->ibdev_path, name, buff, - sizeof(buff)) <= 0) { - char *dir_path; - DIR *dir; - - if (errno == EINVAL) { - /* In IB, this file doesn't exist and the kernel sets - * errno to -EINVAL. - */ - *type = IBV_GID_TYPE_IB_ROCE_V1; - return 0; - } - if (asprintf(&dir_path, "%s/%s/%d/%s/", - context->device->ibdev_path, "ports", port_num, - "gid_attrs") < 0) - return -1; - dir = opendir(dir_path); - free(dir_path); - if (!dir) { - if (errno == ENOENT) - /* Assuming that if gid_attrs doesn't exist, - * we have an old kernel and all GIDs are - * IB/RoCE v1 - */ - *type = IBV_GID_TYPE_IB_ROCE_V1; - else - return -1; - } else { - closedir(dir); - errno = EFAULT; - return -1; - } - } else { - if (!strcmp(buff, V1_TYPE)) { - *type = IBV_GID_TYPE_IB_ROCE_V1; - } else if (!strcmp(buff, V2_TYPE)) { - *type = IBV_GID_TYPE_ROCE_V2; - } else { - errno = ENOTSUP; - return -1; - } - } - - return 0; -} - -static int ibv_find_gid_index(struct ibv_context *context, uint8_t port_num, - union ibv_gid *gid, enum ibv_gid_type gid_type) -{ - enum ibv_gid_type sgid_type = 0; - union ibv_gid sgid; - int i = 0, ret; - - do { - ret = ibv_query_gid(context, port_num, i, &sgid); - if (!ret) { - ret = ibv_query_gid_type(context, port_num, i, - &sgid_type); - } - i++; - } while (!ret && (memcmp(&sgid, gid, sizeof(*gid)) || - (gid_type != sgid_type))); - - return ret ? ret : i - 1; -} - -static inline void map_ipv4_addr_to_ipv6(__be32 ipv4, struct in6_addr *ipv6) -{ - ipv6->s6_addr32[0] = 0; - ipv6->s6_addr32[1] = 0; - ipv6->s6_addr32[2] = htonl(0x0000FFFF); - ipv6->s6_addr32[3] = ipv4; -} - -static inline uint16_t ipv4_calc_hdr_csum(uint16_t *data, unsigned int num_hwords) -{ - unsigned int i = 0; - uint32_t sum = 0; - - for (i = 0; i < num_hwords; i++) - sum += *(data++); - - sum = (sum & 0xffff) + (sum >> 16); - - return ~sum; -} - -static inline int get_grh_header_version(struct ibv_grh *grh) -{ - int ip6h_version = (ntohl(grh->version_tclass_flow) >> 28) & 0xf; - struct iphdr *ip4h = (struct iphdr *)((void *)grh + 20); - struct iphdr ip4h_checked; - - if (ip6h_version != 6) { - if (ip4h->version == 4) - return 4; - errno = EPROTONOSUPPORT; - return -1; - } - /* version may be 6 or 4 */ - if (ip4h->ihl != 5) /* IPv4 header length must be 5 for RoCE v2. */ - return 6; - /* - * Verify checksum. - * We can't write on scattered buffers so we have to copy to temp - * buffer. - */ - memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked)); - /* Need to set the checksum field (check) to 0 before re-calculating - * the checksum. - */ - ip4h_checked.check = 0; - ip4h_checked.check = ipv4_calc_hdr_csum((uint16_t *)&ip4h_checked, 10); - /* if IPv4 header checksum is OK, believe it */ - if (ip4h->check == ip4h_checked.check) - return 4; - return 6; -} - -static inline void set_ah_attr_generic_fields(struct ibv_ah_attr *ah_attr, - struct ibv_wc *wc, - struct ibv_grh *grh, - uint8_t port_num) -{ - uint32_t flow_class; - - flow_class = ntohl(grh->version_tclass_flow); - ah_attr->grh.flow_label = flow_class & 0xFFFFF; - ah_attr->dlid = wc->slid; - ah_attr->sl = wc->sl; - ah_attr->src_path_bits = wc->dlid_path_bits; - ah_attr->port_num = port_num; -} - -static inline int set_ah_attr_by_ipv4(struct ibv_context *context, - struct ibv_ah_attr *ah_attr, - struct iphdr *ip4h, uint8_t port_num) -{ - union ibv_gid sgid; - int ret; - - /* No point searching multicast GIDs in GID table */ - if (IN_CLASSD(ntohl(ip4h->daddr))) { - errno = EINVAL; - return -1; - } - - map_ipv4_addr_to_ipv6(ip4h->daddr, (struct in6_addr *)&sgid); - ret = ibv_find_gid_index(context, port_num, &sgid, - IBV_GID_TYPE_ROCE_V2); - if (ret < 0) - return ret; - - map_ipv4_addr_to_ipv6(ip4h->saddr, - (struct in6_addr *)&ah_attr->grh.dgid); - ah_attr->grh.sgid_index = (uint8_t) ret; - ah_attr->grh.hop_limit = ip4h->ttl; - ah_attr->grh.traffic_class = ip4h->tos; - - return 0; -} - -#define IB_NEXT_HDR 0x1b -static inline int set_ah_attr_by_ipv6(struct ibv_context *context, - struct ibv_ah_attr *ah_attr, - struct ibv_grh *grh, uint8_t port_num) -{ - uint32_t flow_class; - uint32_t sgid_type; - int ret; - - /* No point searching multicast GIDs in GID table */ - if (grh->dgid.raw[0] == 0xFF) { - errno = EINVAL; - return -1; - } - - ah_attr->grh.dgid = grh->sgid; - if (grh->next_hdr == IPPROTO_UDP) { - sgid_type = IBV_GID_TYPE_ROCE_V2; - } else if (grh->next_hdr == IB_NEXT_HDR) { - sgid_type = IBV_GID_TYPE_IB_ROCE_V1; - } else { - errno = EPROTONOSUPPORT; - return -1; - } - - ret = ibv_find_gid_index(context, port_num, &grh->dgid, - sgid_type); - if (ret < 0) - return ret; - - ah_attr->grh.sgid_index = (uint8_t) ret; - flow_class = ntohl(grh->version_tclass_flow); - ah_attr->grh.hop_limit = grh->hop_limit; - ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; - - return 0; -} - -int ibv_init_ah_from_wc(struct ibv_context *context, uint8_t port_num, - struct ibv_wc *wc, struct ibv_grh *grh, - struct ibv_ah_attr *ah_attr) -{ - int version; - int ret = 0; - - memset(ah_attr, 0, sizeof *ah_attr); - set_ah_attr_generic_fields(ah_attr, wc, grh, port_num); - - if (wc->wc_flags & IBV_WC_GRH) { - ah_attr->is_global = 1; - version = get_grh_header_version(grh); - - if (version == 4) - ret = set_ah_attr_by_ipv4(context, ah_attr, - (struct iphdr *)((void *)grh + 20), - port_num); - else if (version == 6) - ret = set_ah_attr_by_ipv6(context, ah_attr, grh, - port_num); - else - ret = -1; - } - - return ret; -} - -struct ibv_ah *ibv_create_ah_from_wc(struct ibv_pd *pd, struct ibv_wc *wc, - struct ibv_grh *grh, uint8_t port_num) -{ - struct ibv_ah_attr ah_attr; - int ret; - - ret = ibv_init_ah_from_wc(pd->context, port_num, wc, grh, &ah_attr); - if (ret) - return NULL; - - return ibv_create_ah(pd, &ah_attr); -} - -int __ibv_destroy_ah(struct ibv_ah *ah) -{ - return ah->context->ops.destroy_ah(ah); -} -default_symver(__ibv_destroy_ah, ibv_destroy_ah); - -int __ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) -{ - return qp->context->ops.attach_mcast(qp, gid, lid); -} -default_symver(__ibv_attach_mcast, ibv_attach_mcast); - -int __ibv_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) -{ - return qp->context->ops.detach_mcast(qp, gid, lid); -} -default_symver(__ibv_detach_mcast, ibv_detach_mcast); - -static inline int ipv6_addr_v4mapped(const struct in6_addr *a) -{ - return IN6_IS_ADDR_V4MAPPED(&a->s6_addr32) || - /* IPv4 encoded multicast addresses */ - (a->s6_addr32[0] == htonl(0xff0e0000) && - ((a->s6_addr32[1] | - (a->s6_addr32[2] ^ htonl(0x0000ffff))) == 0UL)); -} - -struct peer_address { - void *address; - uint32_t size; -}; - -static inline int create_peer_from_gid(int family, void *raw_gid, - struct peer_address *peer_address) -{ - switch (family) { - case AF_INET: - peer_address->address = raw_gid + 12; - peer_address->size = 4; - break; - case AF_INET6: - peer_address->address = raw_gid; - peer_address->size = 16; - break; - default: - return -1; - } - - return 0; -} - -#define NEIGH_GET_DEFAULT_TIMEOUT_MS 3000 -int ibv_resolve_eth_l2_from_gid(struct ibv_context *context, - struct ibv_ah_attr *attr, - uint8_t eth_mac[ETHERNET_LL_SIZE], - uint16_t *vid) -{ -#ifndef NRESOLVE_NEIGH - int dst_family; - int src_family; - int oif; - struct get_neigh_handler neigh_handler; - union ibv_gid sgid; - int ether_len; - struct peer_address src; - struct peer_address dst; - uint16_t ret_vid; - int ret = -EINVAL; - int err; - - err = ibv_query_gid(context, attr->port_num, - attr->grh.sgid_index, &sgid); - - if (err) - return err; - - err = neigh_init_resources(&neigh_handler, - NEIGH_GET_DEFAULT_TIMEOUT_MS); - - if (err) - return err; - - dst_family = ipv6_addr_v4mapped((struct in6_addr *)attr->grh.dgid.raw) ? - AF_INET : AF_INET6; - src_family = ipv6_addr_v4mapped((struct in6_addr *)sgid.raw) ? - AF_INET : AF_INET6; - - if (create_peer_from_gid(dst_family, attr->grh.dgid.raw, &dst)) - goto free_resources; - - if (create_peer_from_gid(src_family, &sgid.raw, &src)) - goto free_resources; - - if (neigh_set_dst(&neigh_handler, dst_family, dst.address, - dst.size)) - goto free_resources; - - if (neigh_set_src(&neigh_handler, src_family, src.address, - src.size)) - goto free_resources; - - oif = neigh_get_oif_from_src(&neigh_handler); - - if (oif > 0) - neigh_set_oif(&neigh_handler, oif); - else - goto free_resources; - - ret = -EHOSTUNREACH; - - /* blocking call */ - if (process_get_neigh(&neigh_handler)) - goto free_resources; - - ret_vid = neigh_get_vlan_id_from_dev(&neigh_handler); - - if (ret_vid <= 0xfff) - neigh_set_vlan_id(&neigh_handler, ret_vid); - - /* We are using only Ethernet here */ - ether_len = neigh_get_ll(&neigh_handler, - eth_mac, - sizeof(uint8_t) * ETHERNET_LL_SIZE); - - if (ether_len <= 0) - goto free_resources; - - *vid = ret_vid; - - ret = 0; - -free_resources: - neigh_free_resources(&neigh_handler); - - return ret; -#else - return -ENOSYS; -#endif -} |