ON-12116: ulhelper: drop stack lock from terminate.c

Implement a (slow) way to drop stack lock from kernel context,
passing the lock ownership to the onload_helper process.

src/driver/linux_onload/terminate.c

@@ -268,7 +268,13 @@ efab_terminate_unlock_all_stacks(tcp_helper_resource_t *stacks[],
   for( i = 0; i < stacks_num; i++ ) {
     if( stacks[i] == NULL )
       continue;
+#if CI_CFG_UL_INTERRUPT_HELPER
+    ci_assert_nflags(stacks[i]->ulh_flags, OO_ULH_WAIT_FLAG_LOCKED);
+    ci_atomic32_or(&stacks[i]->ulh_flags, OO_ULH_WAIT_FLAG_LOCKED);
+    wake_up(&stacks[i]->ulh_waitq);
+#else
     ci_netif_unlock(&stacks[i]->netif);
+#endif
     TERM_DEBUG("%s: %d unlock stack %d", __func__, current->pid, stacks[i]->id);
     oo_thr_ref_drop(stacks[i]->ref, OO_THR_REF_BASE);
   }

src/include/onload/common.h

@@ -361,6 +361,10 @@ struct oo_op_loopback_connect {
 
 #if CI_CFG_UL_INTERRUPT_HELPER
 struct oo_ulh_waiter {
+  ci_uint32 flags CI_ALIGN(8);
+  /* Stack is already locked, the helper should release the stack lock */
+#define OO_ULH_WAIT_FLAG_LOCKED 1
+
   ci_uint32 timeout_ms;     /* in */
   ci_uint32 rs_ref_count;   /* out */
 };

src/include/onload/tcp_helper.h

@@ -333,6 +333,8 @@ typedef struct tcp_helper_resource_s {
   ci_atomic_t intr_intfs;
   /* intfs to request wakeup */
   ci_atomic_t wake_intfs;
+
+  ci_uint32 ulh_flags;
 #endif
 
   struct tcp_helper_nic      nic[CI_CFG_MAX_INTERFACES];

src/lib/efthrm/tcp_helper_resource.c

@@ -2164,6 +2164,7 @@ allocate_netif_resources(ci_resource_onload_alloc_t* alloc,
 
 #if CI_CFG_UL_INTERRUPT_HELPER
   init_waitqueue_head(&trs->ulh_waitq);
+  trs->ulh_flags = 0;
 
   { CI_BUILD_ASSERT(CI_CFG_MAX_INTERFACES<=32); }
   ci_atomic_set(&trs->intr_intfs, 0);
@@ -3499,6 +3500,19 @@ stack_has_events(ci_netif* ni, ci_uint32 intfs)
   return false;
 }
 
+static bool
+stack_has_ul_job(tcp_helper_resource_t* trs, ci_uint32* intfs, ci_uint32* flags)
+{
+  *intfs = ci_atomic_xchg(&trs->intr_intfs, 0);
+  do {
+    *flags = trs->ulh_flags;
+    if( *flags == 0 )
+      break;
+  } while( ci_cas32u_fail(&trs->ulh_flags, *flags, 0) );
+
+  return *intfs != 0 || *flags != 0;
+}
+
 static void
 request_pending_wakeups(tcp_helper_resource_t* trs)
 {
@@ -3530,7 +3544,7 @@ int oo_wait_for_interrupt(ci_private_t* priv, void* argp)
     ci_assert_ge(timeout, 1);
     rc = wait_event_interruptible_timeout(
                   trs->ulh_waitq,
-                  (intfs = ci_atomic_xchg(&trs->intr_intfs, 0)) != 0,
+                  stack_has_ul_job(trs, &intfs, &arg->flags),
                   timeout);
     if( NI_OPTS(&trs->netif).int_driven && intfs != 0 )
       ci_atomic_or(&trs->wake_intfs, intfs);
@@ -3541,7 +3555,7 @@ int oo_wait_for_interrupt(ci_private_t* priv, void* argp)
     if( (arg->rs_ref_count = trs->ref[OO_THR_REF_APP]) > 1 )
       break;
     /* interrupts were not handled by the app, do it by the helper */
-    if( stack_has_events(&trs->netif, intfs) )
+    if( stack_has_events(&trs->netif, intfs) || arg->flags != 0 )
       break;
     /* No interrupts, but the stack was not touched for too long.
      * Fixme: Ideally, we'd like to ensure there are no pending ip timers,

src/tools/onload_helper/main.c

@@ -110,8 +110,11 @@ main_loop(ci_netif* ni)
   arg.timeout_ms = 0;
 
   while( ioctl(ni->driver_handle, OO_IOC_WAIT_FOR_INTERRUPT, &arg) == 0 ) {
-    ci_assert( ! is_locked );
-    if( ci_netif_trylock(ni) ) {
+    if( arg.flags & OO_ULH_WAIT_FLAG_LOCKED )
+      is_locked = true;
+    else
+      ci_assert( ! is_locked );
+    if( is_locked || ci_netif_trylock(ni) ) {
       int n = ci_netif_poll(ni);
       CITP_STATS_NETIF_ADD(ni, interrupt_evs, n);
       is_locked = true;
@@ -146,6 +149,7 @@ main_loop(ci_netif* ni)
       is_locked = false;
     }
     arg.timeout_ms = stack_next_timer_ms(ni);
+    ci_assert( ! is_locked );
   }
 }