ix_txrx.c.diff

cvs -q diff -aU30 -a -p ix_txrx.c
Index: ix_txrx.c
===================================================================
RCS file: /src/cvs/cvsroot-netbsd/src/sys/dev/pci/ixgbe/ix_txrx.c,v
retrieving revision 1.72
diff -a -U 30 -a -p -r1.72 ix_txrx.c
--- ix_txrx.c	11 May 2021 01:30:30 -0000	1.72
+++ ix_txrx.c	12 May 2021 06:32:54 -0000
@@ -1307,121 +1307,156 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr)
 	IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
 	    (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR));
 
 	/* Disable RSC for ACK packets */
 	IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
 	    (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
 
 	rxr->hw_rsc = TRUE;
 } /* ixgbe_setup_hw_rsc */
 
 /************************************************************************
  * ixgbe_refresh_mbufs
  *
  *   Refresh mbuf buffers for RX descriptor rings
  *    - now keeps its own state so discards due to resource
  *      exhaustion are unnecessary, if an mbuf cannot be obtained
  *      it just returns, keeping its placeholder, thus it can simply
  *      be recalled to try again.
  *
  *   XXX NetBSD TODO:
  *    - The ixgbe_rxeof() function always preallocates mbuf cluster (jcl),
  *      so the ixgbe_refresh_mbufs() function can be simplified.
  *
  ************************************************************************/
 static void
 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
 {
 	struct adapter      *adapter = rxr->adapter;
 	struct ixgbe_rx_buf *rxbuf;
 	struct mbuf         *mp;
-	int                 i, j, error;
+	int                 i, j, error, idx_start, idx_nsync = 0;
 	bool                refreshed = false;
 
-	i = j = rxr->next_to_refresh;
-	/* Control the loop with one beyond */
+	/*
+	 * we go to refresh mbufs and sync descriptor
+	 * for each cache line size.
+	 * e.g.) dmamap_sync(desc[0..3]), dmamap_sync(desc[4..7]), ...
+	 */
+#define RX_DESC_SIZE		sizeof(union ixgbe_adv_rx_desc)
+#define NDESC_PER_CACHELINE	(CACHE_LINE_SIZE / RX_DESC_SIZE)
+	if ((limit & (NDESC_PER_CACHELINE - 1)) != 1) /* '1' for 'one beyond' */
+		return;
+
+	i = j = idx_start = rxr->next_to_refresh;
+	/*
+	 * To prevent TAIL from catching up with HEAD,
+	 * control the loop with one beyond
+	 */
 	if (++j == rxr->num_desc)
 		j = 0;
 
 	while (j != limit) {
 		rxbuf = &rxr->rx_buffers[i];
 		if (rxbuf->buf == NULL) {
 			mp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT,
 			    MT_DATA, M_PKTHDR, rxr->mbuf_sz);
 			if (mp == NULL) {
 				rxr->no_jmbuf.ev_count++;
 				goto update;
 			}
 			if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
 				m_adj(mp, ETHER_ALIGN);
 		} else
 			mp = rxbuf->buf;
 
 		mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
 
 		/* If we're dealing with an mbuf that was copied rather
 		 * than replaced, there's no need to go through busdma.
 		 */
 		if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
 			/* Get the memory mapping */
 			ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap);
 			error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat,
 			    rxbuf->pmap, mp, BUS_DMA_NOWAIT);
 			if (error != 0) {
 				device_printf(adapter->dev, "Refresh mbufs: "
 				    "payload dmamap load failure - %d\n",
 				    error);
 				m_free(mp);
 				rxbuf->buf = NULL;
 				goto update;
 			}
 			rxbuf->buf = mp;
 			bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap,
 			    0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD);
 			rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
 			    htole64(rxbuf->pmap->dm_segs[0].ds_addr);
 		} else {
 			rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
 			rxbuf->flags &= ~IXGBE_RX_COPY;
 		}
 
+		rxr->rx_base[i].wb.upper.status_error = 0;	/* clear DD */
+		idx_nsync++;
+
 		refreshed = true;
 		/* Next is precalculated */
 		i = j;
 		rxr->next_to_refresh = i;
 		if (++j == rxr->num_desc)
 			j = 0;
+
+		if (i == 0) {
+			bus_dmamap_sync(rxr->rxdma.dma_tag->dt_dmat,
+			    rxr->rxdma.dma_map,
+			    idx_start * sizeof(union ixgbe_adv_rx_desc),
+			    idx_nsync * sizeof(union ixgbe_adv_rx_desc),
+			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+			idx_start = idx_nsync = 0;
+		}
 	}
 
 update:
-	if (refreshed) /* Update hardware tail index */
+	if (idx_nsync > 0) {
+		bus_dmamap_sync(rxr->rxdma.dma_tag->dt_dmat,
+		    rxr->rxdma.dma_map,
+		    idx_start * sizeof(union ixgbe_adv_rx_desc),
+		    idx_nsync * sizeof(union ixgbe_adv_rx_desc),
+		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	}
+
+	if (refreshed) { /* Update hardware tail index */
+		IXGBE_WRITE_BARRIER(&adapter->hw);
 		IXGBE_WRITE_REG(&adapter->hw, rxr->tail, rxr->next_to_refresh);
+	}
 
 	return;
 } /* ixgbe_refresh_mbufs */
 
 /************************************************************************
  * ixgbe_allocate_receive_buffers
  *
  *   Allocate memory for rx_buffer structures. Since we use one
  *   rx_buffer per received packet, the maximum number of rx_buffer's
  *   that we'll need is equal to the number of receive descriptors
  *   that we've allocated.
  ************************************************************************/
 static int
 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
 {
 	struct adapter      *adapter = rxr->adapter;
 	device_t            dev = adapter->dev;
 	struct ixgbe_rx_buf *rxbuf;
 	int                 bsize, error;
 
 	bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
 	rxr->rx_buffers = malloc(bsize, M_DEVBUF, M_WAITOK | M_ZERO);
 
 	error = ixgbe_dma_tag_create(
 	         /*      parent */ adapter->osdep.dmat,
 	         /*   alignment */ 1,
 	         /*      bounds */ 0,
 	         /*     maxsize */ MJUM16BYTES,
 	         /*   nsegments */ 1,
 	         /*  maxsegsize */ MJUM16BYTES,
@@ -1814,77 +1849,78 @@ ixgbe_rxeof(struct ix_queue *que)
 #endif
 
 	IXGBE_RX_LOCK(rxr);
 
 #ifdef DEV_NETMAP
 	if (adapter->feat_en & IXGBE_FEATURE_NETMAP) {
 		/* Same as the txeof routine: wakeup clients on intr. */
 		if (netmap_rx_irq(ifp, rxr->me, &processed)) {
 			IXGBE_RX_UNLOCK(rxr);
 			return (FALSE);
 		}
 	}
 #endif /* DEV_NETMAP */
 
 	/*
 	 * The max number of loop is rx_process_limit. If discard_multidesc is
 	 * true, continue processing to not to send broken packet to the upper
 	 * layer.
 	 */
 	for (i = rxr->next_to_check;
 	     (count < limit) || (discard_multidesc == true);) {
 
 		struct mbuf *sendmp, *mp;
 		struct mbuf *newmp;
 		u32         rsc, ptype;
 		u16         len;
 		u16         vtag = 0;
 		bool        eop;
 
 		/* Sync the ring. */
-		ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+		bus_dmamap_sync(rxr->rxdma.dma_tag->dt_dmat, rxr->rxdma.dma_map,
+		    i * sizeof(union ixgbe_adv_rx_desc),
+		    sizeof(union ixgbe_adv_rx_desc),
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 		cur = &rxr->rx_base[i];
 		staterr = le32toh(cur->wb.upper.status_error);
 #ifdef RSS
 		pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
 #endif
 
 		if ((staterr & IXGBE_RXD_STAT_DD) == 0)
 			break;
 
 		count++;
 		sendmp = NULL;
 		nbuf = NULL;
 		rsc = 0;
-		cur->wb.upper.status_error = 0;
 		rbuf = &rxr->rx_buffers[i];
 		mp = rbuf->buf;
 
 		len = le16toh(cur->wb.upper.length);
 		ptype = le32toh(cur->wb.lower.lo_dword.data) &
 		    IXGBE_RXDADV_PKTTYPE_MASK;
 		eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
 
 		/* Make sure bad packets are discarded */
 		if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
 #if __FreeBSD_version >= 1100036
 			if (adapter->feat_en & IXGBE_FEATURE_VF)
 				if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
 #endif
 			rxr->rx_discarded.ev_count++;
 			ixgbe_rx_discard(rxr, i);
 			discard_multidesc = false;
 			goto next_desc;
 		}
 
 		/* pre-alloc new mbuf */
 		if (!discard_multidesc)
 			newmp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, MT_DATA,
 			    M_PKTHDR, rxr->mbuf_sz);
 		else
 			newmp = NULL;
 		if (newmp == NULL) {
 			rxr->no_jmbuf.ev_count++;
 			/*
 			 * Descriptor initialization is already done by the
@@ -2045,82 +2081,71 @@ ixgbe_rxeof(struct ix_queue *que)
 					break;
 				case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX:
 					M_HASHTYPE_SET(sendmp,
 					    M_HASHTYPE_RSS_TCP_IPV6_EX);
 					break;
 #if __FreeBSD_version > 1100000
 				case IXGBE_RXDADV_RSSTYPE_IPV4_UDP:
 					M_HASHTYPE_SET(sendmp,
 					    M_HASHTYPE_RSS_UDP_IPV4);
 					break;
 				case IXGBE_RXDADV_RSSTYPE_IPV6_UDP:
 					M_HASHTYPE_SET(sendmp,
 					    M_HASHTYPE_RSS_UDP_IPV6);
 					break;
 				case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX:
 					M_HASHTYPE_SET(sendmp,
 					    M_HASHTYPE_RSS_UDP_IPV6_EX);
 					break;
 #endif
 				default:
 					M_HASHTYPE_SET(sendmp,
 					    M_HASHTYPE_OPAQUE_HASH);
 				}
 			} else {
 				sendmp->m_pkthdr.flowid = que->msix;
 				M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
 			}
 #endif
 		}
 next_desc:
-		ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
 		/* Advance our pointers to the next descriptor. */
 		if (++i == rxr->num_desc)
 			i = 0;
 
 		/* Now send to the stack or do LRO */
 		if (sendmp != NULL) {
 			ixgbe_rx_input(rxr, ifp, sendmp, ptype);
 		}
 
-		/* Every 8 descriptors we go to refresh mbufs */
-		if (processed == 8) {
-			ixgbe_refresh_mbufs(rxr, i);
-			processed = 0;
-		}
-	}
-
-	/* Refresh any remaining buf structs */
-	if (ixgbe_rx_unrefreshed(rxr))
 		ixgbe_refresh_mbufs(rxr, i);
+	}
 
 	rxr->next_to_check = i;
 
 	IXGBE_RX_UNLOCK(rxr);
 
 #ifdef LRO
 	/*
 	 * Flush any outstanding LRO work
 	 */
 	tcp_lro_flush_all(lro);
 #endif /* LRO */
 
 	/*
 	 * Still have cleaning to do?
 	 */
 	if ((staterr & IXGBE_RXD_STAT_DD) != 0)
 		return (TRUE);
 
 	return (FALSE);
 } /* ixgbe_rxeof */
 
 
 /************************************************************************
  * ixgbe_rx_checksum
  *
  *   Verify that the hardware indicated that the checksum is valid.
  *   Inform the stack about the status of checksum so that stack
  *   doesn't spend time verifying the checksum.
  ************************************************************************/
 static void