ON-11184: add support for generic network devices with AF_XDP - PoC

Change-Id: I4f67c2dee70ab7d1d707f4123e307fa758ef18d3

scripts/jenkins/ut_pipeline.groovy

@@ -59,7 +59,8 @@ void doORMPipeline(List gcovr_options)
 }
 
 void doDeveloperBuild(String build_profile=null) {
-  def components = ['kernel_driver', 'userspace', 'userspace_32']
+  // TODO: ON-12091 Fix 32 bit builds before merging into master
+  def components = ['kernel_driver', 'userspace']
   def debugnesses = ['DEBUG', 'NDEBUG']
 
   def stage_name = 'Developer Build'

src/driver/linux_char/vi_resource.c

@@ -15,6 +15,7 @@
 #include "filter_list.h"
 #include "linux_char_internal.h"
 
+
 /* Reserved space in evq for a reasonable number of time sync events.
  * They arrive at a rate of 4 per second.  This allows app to get
  * 25s behind...
@@ -265,6 +266,11 @@ efch_vi_rm_alloc(ci_resource_alloc_t* alloc, ci_resource_table_t* rt,
   if( rmpd != NULL && efrm_pd_stack_id_get(rmpd) > 0 )
     in_flags |= EFHW_VI_TX_LOOPBACK;
 
+  efrm_vi_attr_set_af_xdp(&attr,
+                          alloc_in->xdp_buffers,
+                          alloc_in->xdp_buffer_size,
+                          alloc_in->xdp_headroom);
+
   rc = vi_resource_alloc(&attr, client, evq ? efrm_vi(evq) : NULL,
                          in_flags,
                          alloc_in->evq_capacity,
@@ -296,16 +302,20 @@ efch_vi_rm_alloc(ci_resource_alloc_t* alloc, ci_resource_table_t* rt,
   alloc_out = &alloc->u.vi_out;
   CI_DEBUG(alloc = NULL);
   CI_DEBUG(alloc_in = NULL);
+
+  nic = efrm_client_get_nic(virs->rs.rs_client);
   alloc_out->instance = virs->rs.rs_instance;
   alloc_out->evq_capacity = virs->q[EFHW_EVQ].capacity;
   alloc_out->rxq_capacity = virs->q[EFHW_RXQ].capacity;
   alloc_out->txq_capacity = virs->q[EFHW_TXQ].capacity;
-  nic = efrm_client_get_nic(virs->rs.rs_client);
   alloc_out->nic_arch = nic->devtype.arch;
   alloc_out->nic_variant = nic->devtype.variant;
   alloc_out->nic_revision = nic->devtype.revision;
   alloc_out->nic_flags = efhw_vi_nic_flags(nic);
-  alloc_out->io_mmap_bytes = 4096;
+  if (nic->devtype.arch == EFHW_ARCH_AF_XDP)
+    alloc_out->io_mmap_bytes = 0;
+  else
+    alloc_out->io_mmap_bytes = 4096;
   alloc_out->mem_mmap_bytes = virs->mem_mmap_bytes;
   alloc_out->rx_prefix_len = virs->rx_prefix_len;
   alloc_out->out_flags = virs->out_flags;
@@ -547,6 +557,13 @@ efch_vi_rm_rsops(efch_resource_t* rs, ci_resource_table_t* rt,
       *copy_out = 1;
       break;
 
+    case CI_RSOP_VI_AF_XDP_KICK:
+      {
+        struct msghdr msg = {.msg_flags = MSG_DONTWAIT};
+        rc = sock_sendmsg(virs->af_xdp_sock, &msg);
+        break;
+      }
+
     default:
       rc = efch_filter_list_op_add(rs->rs_base, efrm_vi_get_pd(virs),
                                    &rs->vi.fl, op, copy_out, 0u, -1);

src/driver/linux_char/vi_resource_mmap.c

@@ -44,6 +44,8 @@ efab_vi_rm_mmap_io(struct efrm_vi *virs,
   struct efhw_nic *nic;
 
   nic = efrm_client_get_nic(virs->rs.rs_client);
+  if( nic->devtype.arch == EFHW_ARCH_AF_XDP )
+    return 0;
 
   instance = virs->rs.rs_instance;
 
@@ -224,11 +226,13 @@ int
 efab_vi_resource_mmap_bytes(struct efrm_vi* virs, int map_type)
 {
   int bytes = 0;
+  struct efhw_nic *nic = efrm_client_get_nic(virs->rs.rs_client);
 
   EFRM_RESOURCE_ASSERT_VALID(&virs->rs, 0);
 
   if( map_type == 0 ) {  /* I/O mapping. */
-    bytes += CI_PAGE_SIZE;
+    if( nic->devtype.arch != EFHW_ARCH_AF_XDP )
+      bytes += CI_PAGE_SIZE;
   }
   else {              /* Memory mapping. */
     if( virs->q[EFHW_EVQ].capacity != 0 )

src/driver/linux_onload/driver.c

@@ -539,6 +539,8 @@ static int __init onload_module_init(void)
   if( rc != 0 )
     goto fail_ip_ctor;
 
+  oo_nondl_register();
+
   rc = ci_install_proc_entries();
   if( rc < 0 ) {
     ci_log("%s: ERROR: ci_install_proc_entries failed (%d)", __FUNCTION__, rc);
@@ -599,6 +601,7 @@ static int __init onload_module_init(void)
  failed_trampoline:
   ci_uninstall_proc_entries();
  fail_proc:
+  oo_nondl_unregister();
   /* Remove all NICs.
    * It is possible that efx_dl_register_driver() call was successful, so
    * we have to shut down all NICs even if oo_driverlink_register() failed. */
@@ -641,6 +644,7 @@ static void onload_module_exit(void)
 
   /* Remove the rest of external interfaces to efab_tcp_driver. */
   ci_uninstall_proc_entries();
+  oo_nondl_unregister();
 
   /* Remove all NICs when driverlink is gone. */
   oo_nic_shutdown();

src/driver/linux_onload/driverlink_ip.c

@@ -216,6 +216,23 @@ static struct nf_hook_ops oo_netfilter_ip6_hook = {
 #endif
 
 
+int oo_netdev_is_dl(const struct net_device *net_dev)
+{
+  struct efhw_nic* efhw_nic;
+  struct oo_nic* onic;
+  onic = oo_nic_find_dev(net_dev);
+  if( onic == NULL )
+    return 0;
+  efhw_nic = efrm_client_get_nic(onic->efrm_client);
+  return efhw_nic->devtype.arch == EFHW_ARCH_EF10 ||
+         efhw_nic->devtype.arch == EFHW_ARCH_EF100;
+}
+
+int oo_netdev_is_non_dl(const struct net_device *net_dev)
+{
+  return ! oo_netdev_is_dl(net_dev) && oo_nic_find_dev(net_dev) != NULL;
+}
+
 /* This function will create an oo_nic if one hasn't already been created.
  *
  * There are two code paths whereby this function can be called multiple
@@ -225,10 +242,10 @@ static struct nf_hook_ops oo_netfilter_ip6_hook = {
  *   oo_netdev_event() will call oo_netdev_may_add() before dl_probe is run,
  *   which will call oo_netdev_may_add() itself.
  *
- * Once a device is noticed by onload, it should stay registered in cplane 
+ * Once a device is noticed by onload, it should stay registered in cplane
  * despite going up or being hotplugged.
  */
-static struct oo_nic *oo_netdev_may_add(const struct net_device *net_dev)
+struct oo_nic *oo_netdev_may_add(const struct net_device *net_dev)
 {
   struct efhw_nic* efhw_nic;
   struct oo_nic* onic;
@@ -269,7 +286,10 @@ static int oo_dl_probe(struct efx_dl_device* dl_dev,
                        const char* silicon_rev)
 {
   struct oo_nic* onic = NULL;
-
+  if( oo_netdev_is_non_dl(net_dev) ) {
+    ci_log("%s: net dev event %s ignoring as already non-dl", __FUNCTION__, net_dev->name);
+    return -1;
+  }
   if( ! netif_running(net_dev) ) {
     onic = oo_nic_find_dev(net_dev);
     if( onic != NULL ) {
@@ -299,8 +319,7 @@ static int oo_dl_probe(struct efx_dl_device* dl_dev,
   return 0;
 }
 
-
-static void oo_dl_remove(struct efx_dl_device* dl_dev)
+void oo_common_remove(struct net_device* netdev)
 {
   /* We need to fini all of the hardware queues immediately. The net driver
    * will tidy up its own queues and *all* VIs, so if we don't free our own
@@ -315,7 +334,6 @@ static void oo_dl_remove(struct efx_dl_device* dl_dev)
 #if CI_CFG_NIC_RESET_SUPPORT
   ci_netif* ni = NULL;
 #endif
-  struct net_device* netdev = dl_dev->priv;
   struct oo_nic* onic;
   if( (onic = oo_nic_find_dev(netdev)) != NULL ) {
     /* Filter status need to be synced as after this function is finished
@@ -345,6 +363,16 @@ static void oo_dl_remove(struct efx_dl_device* dl_dev)
   }
 }
 
+static void oo_dl_remove(struct efx_dl_device* dl_dev)
+{
+  struct net_device* netdev = dl_dev->priv;
+  if( ! oo_netdev_is_dl(netdev) ) {
+    ci_log("%s: net dev event %s ignoring as non-dl", __FUNCTION__, netdev->name);
+    return;
+  }
+  oo_common_remove(netdev);
+}
+
 
 static void oo_dl_reset_suspend(struct efx_dl_device* dl_dev)
 {
@@ -381,7 +409,7 @@ static void oo_fixup_wakeup_breakage(struct net_device* dev)
 }
 
 
-static void oo_netdev_up(struct net_device* netdev)
+void oo_netdev_up(struct net_device* netdev)
 {
   struct oo_nic *onic;
   struct efhw_nic* efhw_nic;
@@ -420,6 +448,10 @@ static int oo_netdev_event(struct notifier_block *this,
                            unsigned long event, void *ptr)
 {
   struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
+  if( ! oo_netdev_is_dl(netdev) ) {
+    ci_log("%s: net dev event %s ignoring as non-dl", __FUNCTION__, netdev->name);
+    return NOTIFY_DONE;
+  }
 
   switch( event ) {
   case NETDEV_UP:

src/driver/linux_onload/mmake.mk

@@ -14,7 +14,7 @@ ONLOAD_SRCS	:= driver.c timesync.c \
 		linux_trampoline.c shmbuf.c compat.c \
 		ossock_calls.c mmap.c \
 		epoll_device.c terminate.c sigaction_calls.c onloadfs.c \
-		dshm.c cplane.c cplane_prot.c
+		dshm.c onload_nondl.c cplane.c cplane_prot.c
 
 # This is a kernel makefile, so gets re-called by kbuild with 'src' set
 src ?= .

src/driver/linux_onload/mmap.c

@@ -374,6 +374,8 @@ tcp_helper_rm_nopage(tcp_helper_resource_t* trs, struct vm_area_struct *vma,
       /* IO mappings are always present, and so a page fault should never come
        * down this path, but ptrace() can get us here. */
       return NULL;
+    case CI_NETIF_MMAP_ID_PKTS_ALL:
+      return tcp_helper_rm_nopage_pkts(trs, vma, offset);
     default:
       ci_assert_ge(map_id, CI_NETIF_MMAP_ID_PKTS);
       return tcp_helper_rm_nopage_pkts(trs, vma,
@@ -609,14 +611,14 @@ static int tcp_helper_rm_mmap_buf(tcp_helper_resource_t* trs,
   return 0;
 }
 
-/* fixme: this handler is linux-only */
-static int tcp_helper_rm_mmap_pkts(tcp_helper_resource_t* trs,
-                                   unsigned long bytes,
-                                   struct vm_area_struct* vma, int map_id)
+
+static int __tcp_helper_rm_mmap_pkts(tcp_helper_resource_t* trs,
+                                     unsigned long start, unsigned long bytes,
+                                     struct vm_area_struct* vma, int bufid,
+                                     int force_faulting)
 {
   ci_netif* ni;
   ci_netif_state* ns;
-  int bufid = map_id - CI_NETIF_MMAP_ID_PKTS;
 
   if( bytes != CI_CFG_PKT_BUF_SIZE * PKTS_PER_SET )
     return -EINVAL;
@@ -639,16 +641,59 @@ static int tcp_helper_rm_mmap_pkts(tcp_helper_resource_t* trs,
   }
 #endif
 
-  if( oo_iobufset_npages(ni->pkt_bufs[bufid]) == 1 ) {
+  if( oo_iobufset_npages(ni->pkt_bufs[bufid]) == 1 && !force_faulting ) {
     /* Avoid nopage handler, mmap it all at once */
-    return remap_pfn_range(vma, vma->vm_start,
+    return remap_pfn_range(vma, start,
                            oo_iobufset_pfn(ni->pkt_bufs[bufid], 0), bytes,
                            vma->vm_page_prot);
   }
 
   return 0;
 }
 
+static int tcp_helper_rm_mmap_pkts(tcp_helper_resource_t* trs,
+                                   unsigned long bytes,
+                                   struct vm_area_struct* vma, int map_id)
+{
+  int bufid = map_id - CI_NETIF_MMAP_ID_PKTS;
+
+  if( bytes != CI_CFG_PKT_BUF_SIZE * PKTS_PER_SET )
+    return -EINVAL;
+
+  return __tcp_helper_rm_mmap_pkts(trs, vma->vm_start, bytes, vma, bufid, false);
+}
+
+
+static int tcp_helper_rm_mmap_pkts_all(tcp_helper_resource_t* trs,
+                                       unsigned long bytes,
+                                       struct vm_area_struct* vma)
+{
+  ci_netif* ni;
+  int bufid;
+  int rc = 0;
+  unsigned long pkt_buf_size = CI_CFG_PKT_BUF_SIZE * PKTS_PER_SET;
+
+  ni = &trs->netif;
+
+  if( bytes != pkt_buf_size * ni->packets->sets_max )
+    return -EINVAL;
+
+  for( bufid = 0; bufid < ni->packets->sets_max; ++bufid) {
+    if( ni->pkt_bufs[bufid] == NULL )
+      return -EINVAL;
+  }
+
+  for( bufid = 0; bufid < ni->packets->sets_max; ++bufid) {
+    /* TODO: This mapping is for AF_XDP we need to force page_faulting to
+     * get ensure get_user_pages on packet buffer memory can be performed */
+    rc = __tcp_helper_rm_mmap_pkts(trs, vma->vm_start + pkt_buf_size * bufid,
+                                   pkt_buf_size, vma, bufid, true);
+    if( rc < 0 )
+      break;
+  }
+  return rc;
+}
+
 
 static int
 efab_tcp_helper_rm_mmap(tcp_helper_resource_t* trs, unsigned long bytes,
@@ -684,6 +729,9 @@ efab_tcp_helper_rm_mmap(tcp_helper_resource_t* trs, unsigned long bytes,
     case CI_NETIF_MMAP_ID_IOBUFS:
       rc = tcp_helper_rm_mmap_buf(trs, bytes, vma);
       break;
+    case CI_NETIF_MMAP_ID_PKTS_ALL:
+      rc = tcp_helper_rm_mmap_pkts_all(trs, bytes, vma);
+      break;
     default:
       /* CI_NETIF_MMAP_ID_PKTS + set_id */
       rc = tcp_helper_rm_mmap_pkts(trs, bytes, vma, map_id);

src/driver/linux_onload/onload_internal.h

@@ -9,6 +9,8 @@
 extern int oo_driverlink_register(void);
 extern void oo_driverlink_unregister(void);
 
+extern void oo_nondl_register(void);
+extern void oo_nondl_unregister(void);
 #ifdef EFRM_HAVE_NF_NET_HOOK
 struct net;
 int oo_register_nfhook(struct net *net);