align from FreeBSD

sys/dev/netmap/if_re_netmap.h

@@ -26,7 +26,7 @@
  */
 
 /*
- * $FreeBSD: head/sys/dev/netmap/if_re_netmap.h 234225 2012-04-13 15:33:12Z luigi $
+ * $FreeBSD$
  *
  * netmap support for: re
  *

sys/dev/netmap/if_vtnet_netmap.h

@@ -24,7 +24,7 @@
  */
 
 /*
- * $FreeBSD: head/sys/dev/netmap/if_vtnet_netmap.h 340436 2018-11-14 15:39:48Z vmaffione $
+ * $FreeBSD$
  */
 
 #include <net/netmap.h>
@@ -33,77 +33,24 @@
 #include <vm/pmap.h>    /* vtophys ? */
 #include <dev/netmap/netmap_kern.h>
 
-/*
- * Return 1 if the queue identified by 't' and 'idx' is in netmap mode.
- */
-static int
-vtnet_netmap_queue_on(struct vtnet_softc *sc, enum txrx t, int idx)
-{
-	struct netmap_adapter *na = NA(sc->vtnet_ifp);
-
-	if (!nm_native_on(na))
-		return 0;
-
-	if (t == NR_RX)
-		return !!(idx < na->num_rx_rings &&
-			na->rx_rings[idx]->nr_mode == NKR_NETMAP_ON);
-
-	return !!(idx < na->num_tx_rings &&
-		na->tx_rings[idx]->nr_mode == NKR_NETMAP_ON);
-}
-
 /* Register and unregister. */
 static int
 vtnet_netmap_reg(struct netmap_adapter *na, int state)
 {
 	struct ifnet *ifp = na->ifp;
 	struct vtnet_softc *sc = ifp->if_softc;
-	int success;
-	int i;
-
-	/* Drain the taskqueues to make sure that there are no worker threads
-	 * accessing the virtqueues. */
-	vtnet_drain_taskqueues(sc);
 
+	/*
+	 * Trigger a device reinit, asking vtnet_init_locked() to
+	 * also enter or exit netmap mode.
+	 */
 	VTNET_CORE_LOCK(sc);
-
-	/* We need nm_netmap_on() to return true when called by
-	 * vtnet_init_locked() below. */
-	if (state)
-		nm_set_native_flags(na);
-
-	/* We need to trigger a device reset in order to unexpose guest buffers
-	 * published to the host. */
-	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-	/* Get pending used buffers. The way they are freed depends on whether
-	 * they are netmap buffer or they are mbufs. We can tell apart the two
-	 * cases by looking at kring->nr_mode, before this is possibly updated
-	 * in the loop below. */
-	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
-		struct vtnet_txq *txq = &sc->vtnet_txqs[i];
-		struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
-
-		VTNET_TXQ_LOCK(txq);
-		vtnet_txq_free_mbufs(txq);
-		VTNET_TXQ_UNLOCK(txq);
-
-		VTNET_RXQ_LOCK(rxq);
-		vtnet_rxq_free_mbufs(rxq);
-		VTNET_RXQ_UNLOCK(rxq);
-	}
-	vtnet_init_locked(sc);
-	success = (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO;
-
-	if (state) {
-		netmap_krings_mode_commit(na, state);
-	} else {
-		nm_clear_native_flags(na);
-		netmap_krings_mode_commit(na, state);
-	}
-
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	vtnet_init_locked(sc, state ? VTNET_INIT_NETMAP_ENTER
+	    : VTNET_INIT_NETMAP_EXIT);
 	VTNET_CORE_UNLOCK(sc);
 
-	return success;
+	return (0);
 }
 
 
@@ -196,9 +143,11 @@ vtnet_netmap_txsync(struct netmap_kring *kring, int flags)
 }
 
 /*
- * Publish (up to) num netmap receive buffers to the host,
- * starting from the first one that the user made available
- * (kring->nr_hwcur).
+ * Publish 'num 'netmap receive buffers to the host, starting
+ * from the next available one (rx->vtnrx_nm_refill).
+ * Return a positive error code on error, and 0 on success.
+ * If we could not publish all of the buffers that's an error,
+ * since the netmap ring and the virtqueue would go out of sync.
  */
 static int
 vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num)
@@ -208,7 +157,7 @@ vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num)
 	struct netmap_ring *ring = kring->ring;
 	u_int ring_nr = kring->ring_id;
 	u_int const lim = kring->nkr_num_slots - 1;
-	u_int nm_i = kring->nr_hwcur;
+	u_int nm_i;
 
 	/* device-specific */
 	struct vtnet_softc *sc = ifp->if_softc;
@@ -219,15 +168,16 @@ vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num)
 	struct sglist_seg ss[2];
 	struct sglist sg = { ss, 0, 0, 2 };
 
-	for (; num > 0; nm_i = nm_next(nm_i, lim), num--) {
+	for (nm_i = rxq->vtnrx_nm_refill; num > 0;
+	    nm_i = nm_next(nm_i, lim), num--) {
 		struct netmap_slot *slot = &ring->slot[nm_i];
 		uint64_t paddr;
 		void *addr = PNMB(na, slot, &paddr);
 		int err;
 
 		if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
 			if (netmap_ring_reinit(kring))
-				return -1;
+				return EFAULT;
 		}
 
 		slot->flags &= ~NS_BUF_CHANGED;
@@ -240,14 +190,14 @@ vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num)
 		err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg,
 				/*readable=*/0, /*writeable=*/sg.sg_nseg);
 		if (unlikely(err)) {
-			if (err != ENOSPC)
-				nm_prerr("virtqueue_enqueue(%s) failed: %d",
-					kring->name, err);
+			nm_prerr("virtqueue_enqueue(%s) failed: %d",
+				kring->name, err);
 			break;
 		}
 	}
+	rxq->vtnrx_nm_refill = nm_i;
 
-	return nm_i;
+	return num == 0 ? 0 : ENOSPC;
 }
 
 /*
@@ -261,24 +211,30 @@ vtnet_netmap_rxq_populate(struct vtnet_rxq *rxq)
 {
 	struct netmap_adapter *na = NA(rxq->vtnrx_sc->vtnet_ifp);
 	struct netmap_kring *kring;
+	struct netmap_slot *slot;
 	int error;
+	int num;
 
-	if (!nm_native_on(na) || rxq->vtnrx_id >= na->num_rx_rings)
+	slot = netmap_reset(na, NR_RX, rxq->vtnrx_id, 0);
+	if (slot == NULL)
 		return -1;
-
 	kring = na->rx_rings[rxq->vtnrx_id];
-	if (!(nm_kring_pending_on(kring) ||
-			kring->nr_pending_mode == NKR_NETMAP_ON))
-		return -1;
 
-	/* Expose all the RX netmap buffers we can. In case of no indirect
+	/*
+	 * Expose all the RX netmap buffers we can. In case of no indirect
 	 * buffers, the number of netmap slots in the RX ring matches the
 	 * maximum number of 2-elements sglist that the RX virtqueue can
-	 * accommodate (minus 1 to avoid netmap ring wraparound). */
-	error = vtnet_netmap_kring_refill(kring, na->num_rx_desc - 1);
+	 * accommodate. We need to start from kring->nr_hwtail, which is 0
+	 * on the first netmap register and may be different from 0 if a
+	 * virtio re-init (caused by a netma register or i.e., ifconfig)
+	 * happens while the device is in use by netmap.
+	 */
+	rxq->vtnrx_nm_refill = kring->nr_hwtail;
+	num = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
+	error = vtnet_netmap_kring_refill(kring, num);
 	virtqueue_notify(rxq->vtnrx_vq);
 
-	return error < 0 ? ENXIO : 0;
+	return error;
 }
 
 /* Reconcile kernel and user view of the receive ring. */
@@ -304,8 +260,11 @@ vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
 	/*
 	 * First part: import newly received packets.
 	 * Only accept our own buffers (matching the token). We should only get
-	 * matching buffers. We may need to stop early to avoid hwtail to overrun
-	 * hwcur.
+	 * matching buffers. The hwtail should never overrun hwcur, because
+	 * we publish only N-1 receive buffers (and not N).
+	 * In any case we must not leave this routine with the interrupts
+	 * disabled, pending packets in the VQ and hwtail == (hwcur - 1),
+	 * otherwise the pending packets could stall.
 	 */
 	if (netmap_no_pendintr || force_update) {
 		uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);
@@ -314,10 +273,17 @@ vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
 		vtnet_rxq_disable_intr(rxq);
 
 		nm_i = kring->nr_hwtail;
-		while (nm_i != hwtail_lim) {
+		for (;;) {
 			int len;
 			token = virtqueue_dequeue(vq, &len);
 			if (token == NULL) {
+				/*
+				 * Enable the interrupts again and double-check
+				 * for more work. We can go on until we win the
+				 * race condition, since we are not replenishing
+				 * in the meanwhile, and thus we will process at
+				 * most N-1 slots.
+				 */
 				if (interrupts && vtnet_rxq_enable_intr(rxq)) {
 					vtnet_rxq_disable_intr(rxq);
 					continue;
@@ -327,6 +293,11 @@ vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
 			if (unlikely(token != (void *)rxq)) {
 				nm_prerr("BUG: RX token mismatch");
 			} else {
+				if (nm_i == hwtail_lim) {
+					KASSERT(false, ("hwtail would "
+					    "overrun hwcur"));
+				}
+
 				/* Skip the virtio-net header. */
 				len -= sc->vtnet_hdr_size;
 				if (unlikely(len < 0)) {
@@ -342,28 +313,30 @@ vtnet_netmap_rxsync(struct netmap_kring *kring, int flags)
 		kring->nr_hwtail = nm_i;
 		kring->nr_kflags &= ~NKR_PENDINTR;
 	}
-	nm_prdis("[B] h %d c %d hwcur %d hwtail %d", ring->head, ring->cur,
-				kring->nr_hwcur, kring->nr_hwtail);
 
 	/*
 	 * Second part: skip past packets that userspace has released.
 	 */
 	nm_i = kring->nr_hwcur; /* netmap ring index */
 	if (nm_i != head) {
-		int howmany = head - nm_i;
-		int nm_j;
-
-		if (howmany < 0)
-			howmany += kring->nkr_num_slots;
-		nm_j = vtnet_netmap_kring_refill(kring, howmany);
-		if (nm_j < 0)
-			return nm_j;
-		kring->nr_hwcur = nm_j;
+		int released;
+		int error;
+
+		released = head - nm_i;
+		if (released < 0)
+			released += kring->nkr_num_slots;
+		error = vtnet_netmap_kring_refill(kring, released);
+		if (error) {
+			nm_prerr("Failed to replenish RX VQ with %u sgs",
+			    released);
+			return error;
+		}
+		kring->nr_hwcur = head;
 		virtqueue_notify(vq);
 	}
 
-	nm_prdis("[C] h %d c %d t %d hwcur %d hwtail %d", ring->head, ring->cur,
-		ring->tail, kring->nr_hwcur, kring->nr_hwtail);
+	nm_prdis("h %d c %d t %d hwcur %d hwtail %d", kring->rhead,
+	    kring->rcur, kring->rtail, kring->nr_hwcur, kring->nr_hwtail);
 
 	return 0;
 }

sys/dev/netmap/netmap.c

@@ -619,6 +619,10 @@ netmap_set_all_rings(struct netmap_adapter *na, int stopped)
 	if (!nm_netmap_on(na))
 		return;
 
+	if (netmap_verbose) {
+		nm_prinf("%s: %sable all rings", na->name,
+		    (stopped ? "dis" : "en"));
+	}
 	for_rx_tx(t) {
 		for (i = 0; i < netmap_real_rings(na, t); i++) {
 			netmap_set_ring(na, i, t, stopped);
@@ -4569,7 +4573,9 @@ netmap_init(void)
 	if (error)
 		goto fail;
 
+#if !defined(__FreeBSD__) || defined(KLD_MODULE)
 	nm_prinf("netmap: loaded module");
+#endif
 	return (0);
 fail:
 	netmap_fini();

sys/dev/netmap/netmap_bdg.c

@@ -58,7 +58,7 @@ ports attached to the switch)
 
 #if defined(__FreeBSD__)
 #include <sys/cdefs.h> /* prerequisite */
-__FBSDID("$FreeBSD: head/sys/dev/netmap/netmap.c 257176 2013-10-26 17:58:36Z glebius $");
+__FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/errno.h>

sys/dev/netmap/netmap_generic.c

@@ -67,7 +67,7 @@
 #ifdef __FreeBSD__
 
 #include <sys/cdefs.h> /* prerequisite */
-__FBSDID("$FreeBSD: head/sys/dev/netmap/netmap_generic.c 274353 2014-11-10 20:19:58Z luigi $");
+__FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/errno.h>

sys/dev/netmap/netmap_kern.h

@@ -28,7 +28,7 @@
  */
 
 /*
- * $FreeBSD: head/sys/dev/netmap/netmap_kern.h 238985 2012-08-02 11:59:43Z luigi $
+ * $FreeBSD$
  *
  * The header contains the definitions of constants and function
  * prototypes used only in kernelspace.
@@ -1414,19 +1414,19 @@ nm_native_on(struct netmap_adapter *na)
 static inline struct netmap_kring *
 netmap_kring_on(struct netmap_adapter *na, u_int q, enum txrx t)
 {
-        struct netmap_kring *kring = NULL;
+	struct netmap_kring *kring = NULL;
 
-        if (!nm_native_on(na))
-                return NULL;
+	if (!nm_native_on(na))
+		return NULL;
 
-        if (t == NR_RX && q < na->num_rx_rings)
-                kring = na->rx_rings[q];
-        else if (t == NR_TX && q < na->num_tx_rings)
-                kring = na->tx_rings[q];
-        else
-                return NULL;
+	if (t == NR_RX && q < na->num_rx_rings)
+		kring = na->rx_rings[q];
+	else if (t == NR_TX && q < na->num_tx_rings)
+		kring = na->tx_rings[q];
+	else
+		return NULL;
 
-        return (kring->nr_mode == NKR_NETMAP_ON) ? kring : NULL;
+	return (kring->nr_mode == NKR_NETMAP_ON) ? kring : NULL;
 }
 
 static inline int
@@ -1673,7 +1673,6 @@ int netmap_adapter_put(struct netmap_adapter *na);
 #define NETMAP_BUF_BASE(_na)	((_na)->na_lut.lut[0].vaddr)
 #define NETMAP_BUF_SIZE(_na)	((_na)->na_lut.objsize)
 extern int netmap_no_pendintr;
-extern int netmap_mitigate;
 extern int netmap_verbose;
 #ifdef CONFIG_NETMAP_DEBUG
 extern int netmap_debug;		/* for debugging */
@@ -1695,7 +1694,6 @@ enum {                                  /* debug flags */
 };
 
 extern int netmap_txsync_retry;
-extern int netmap_flags;
 extern int netmap_generic_hwcsum;
 extern int netmap_generic_mit;
 extern int netmap_generic_ringsize;