Merge pull request #131 from garlick/rdma_buildfix

tidy up experimental RDMA feature

.github/workflows/main.yml

@@ -17,8 +17,8 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y \
-            libpopt-dev ncurses-dev automake autoconf pkgconf \
-            lua5.3 liblua5.3-dev libmunge-dev libwrap0-dev libcap-dev \
+            ncurses-dev automake autoconf pkgconf \
+            lua5.3 liblua5.3-dev libmunge-dev libcap-dev \
             libattr1-dev dbench attr scrub valgrind ${{matrix.cc}}
     - name: Display configuration
       run: |
@@ -46,3 +46,22 @@ jobs:
       run: sudo make check -C tests/kern
     - name: make distcheck
       run: make distcheck
+
+  build-rdma:
+    name: build-only check for RDMA (experimental)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+            ncurses-dev automake autoconf pkgconf \
+            lua5.3 liblua5.3-dev libmunge-dev libcap-dev \
+            libattr1-dev libibverbs-dev librdmacm-dev gcc
+    - name: autogen
+      run: ./autogen.sh
+    - name: configure
+      run: ./configure --enable-rdma
+    - name: make
+      run: make

config/x_ac_rdma.m4

@@ -0,0 +1,27 @@
+AC_DEFUN([X_AC_RDMA], [
+
+got_rdma=no
+AC_ARG_ENABLE([rdma],
+  [AS_HELP_STRING([--enable-rdma],
+      [build Infiniband RDMA transport (experimental)])],
+  [want_rdma=yes], [want_rdma=no])
+
+if test x$want_rdma == xyes; then
+  X_AC_CHECK_COND_LIB(rdmacm, rdma_accept)
+  X_AC_CHECK_COND_LIB(ibverbs, ibv_alloc_pd)
+  AC_CHECK_HEADER([infiniband/verbs.h])
+  AC_CHECK_HEADER([rdma/rdma_cma.h])
+  if test x$ac_cv_lib_rdmacm_rdma_accept == xyes -a \
+      x$ac_cv_lib_ibverbs_ibv_alloc_pd == xyes -a \
+      x$ac_cv_header_infiniband_verbs_h == xyes -a \
+      x$ac_cv_header_rdma_rdma_cma_h == xyes; then
+    got_rdma=yes
+    AC_DEFINE([WITH_RDMA], [1], [build Infiniband RDMA transport])
+  else
+    AC_MSG_ERROR([Could not configure RDMA: missing ibverbs/rdmacm packages])
+  fi
+fi
+
+AM_CONDITIONAL([RDMA], [test "x$got_rdma" != xno])
+
+])

configure.ac

@@ -78,7 +78,7 @@ X_AC_CHECK_PTHREADS
 X_AC_CHECK_COND_LIB(munge, munge_ctx_create)
 X_AC_CHECK_COND_LIB(cap, cap_get_proc)
 X_AC_TCMALLOC
-X_AC_RDMATRANS
+X_AC_RDMA
 
 ##
 # For list.c, hostlist.c, hash.c

src/cmd/diod.c

@@ -42,7 +42,7 @@
 #include "src/libdiod/diod_log.h"
 #include "src/libdiod/diod_conf.h"
 #include "src/libdiod/diod_sock.h"
-#if WITH_RDMATRANS
+#if WITH_RDMA
 #include "src/libdiod/diod_rdma.h"
 #endif
 
@@ -344,7 +344,7 @@ struct svc_struct {
     pthread_t t;
     int shutdown;
     int reload;
-#if WITH_RDMATRANS
+#if WITH_RDMA
     diod_rdma_t rdma;
     pthread_t rdma_t;
 #endif
@@ -421,7 +421,7 @@ _service_loop (void *arg)
     return NULL;
 }
 
-#if WITH_RDMATRANS
+#if WITH_RDMA
 static void *
 _service_loop_rdma (void *arg)
 {
@@ -541,10 +541,6 @@ _service_run (srvmode_t mode, int rfdno, int wfdno)
         case SRV_SOCKTEST:
             if (!diod_sock_listen (l, &ss.fds, &ss.nfds))
                 msg_exit ("failed to set up listener");
-#if WITH_RDMATRANS
-            ss.rdma = diod_rdma_create ();
-            diod_rdma_listen (ss.rdma);
-#endif
             break;
     }
 
@@ -603,6 +599,16 @@ _service_run (srvmode_t mode, int rfdno, int wfdno)
 #endif
     }
 
+#if WITH_RDMA
+    /* RDMA needs to be initialized after user transitions.
+     * See chaos/diod#107.
+     */
+    if (mode == SRV_NORMAL) {
+        ss.rdma = diod_rdma_create ();
+        diod_rdma_listen (ss.rdma);
+    }
+#endif
+
     /* Process dumpable flag may have been cleared by uid manipulation above.
      * Set it here, then maintain it in user.c::np_setfsid () as uids are
      * further manipulated.
@@ -621,7 +627,7 @@ _service_run (srvmode_t mode, int rfdno, int wfdno)
 
     if ((n = pthread_create (&ss.t, NULL, _service_loop, NULL)))
         errn_exit (n, "pthread_create _service_loop");
-#if WITH_RDMATRANS
+#if WITH_RDMA
     if ((n = pthread_create (&ss.rdma_t, NULL, _service_loop_rdma, NULL)))
         errn_exit (n, "pthread_create _service_loop_rdma");
 #endif
@@ -636,7 +642,7 @@ _service_run (srvmode_t mode, int rfdno, int wfdno)
     }
     if ((n = pthread_join (ss.t, NULL)))
         errn_exit (n, "pthread_join _service_loop");
-#if WITH_RDMATRANS
+#if WITH_RDMA
     if ((n = pthread_join (ss.rdma_t, NULL)))
         errn_exit (n, "pthread_join _service_loop_rdma");
 #endif

src/libdiod/Makefile.am

@@ -27,7 +27,7 @@ libdiod_a_SOURCES = \
 	diod_ops.h \
 	lsderr.c
 
-if RDMATRANS
+if RDMA
 libdiod_a_SOURCES += diod_rdma.c diod_rdma.h
 endif
 

src/libdiod/diod_rdma.c

@@ -35,8 +35,8 @@
 #include <infiniband/verbs.h>
 #include <rdma/rdma_cma.h>
 
-#include "npfs.h"
-#include "list.h"
+#include "src/libnpfs/npfs.h"
+#include "src/liblsd/list.h"
 
 #include "diod_log.h"
 #include "diod_rdma.h"

src/libnpfs/Makefile.am

@@ -34,7 +34,7 @@ libnpfs_a_SOURCES += user-stub.c
 endif
 endif
 
-if RDMATRANS
+if RDMA
 libnpfs_a_SOURCES += rdmatrans.c
 endif