Merge branch 'eth-fbnic-fix-xdp_tx-and-xdp-vs-qstats'

Jakub Kicinski says:

====================
eth: fbnic: fix XDP_TX and XDP vs qstats

Fix XDP_TX hangs and adjust the XDP statistics to match the definition
of qstats. The three problems are somewhat distinct.

XDP_TX hangs is a simple coding bug (patch 1).

The accounting of XDP packets is all over the place. Fix it to obey
qstat rules (packets seen by XDP always counted as Rx packets).
Patch 2 fixes the basic accounting, patch 3 touches up saving
the stats when rings are freed.

Patch 6 corrects reporting of alloc_fail stats which prevented
the pp_alloc_fail test from passing.

Patches 4, 5, 7, 8, 9 add or fix related test cases.

v2:
 - [patch 2] remove now unnecessary byte adjustment
 - [patch 8] use seen_fails more
v1: https://lore.kernel.org/20251003233025.1157158-1-kuba@kernel.org

Testing on fbnic below:

 $ ./tools/testing/selftests/drivers/net/hw/pp_alloc_fail.py
 TAP version 13
 1..1
 fbnic-err: bad MMIO read address 0x80074
 fbnic-err: bad MMIO read address 0x80074
 # Seen: pkts:20605 fails:40 (pass thrs:12)
 # ethtool -G change retval: success
 ok 1 pp_alloc_fail.test_pp_alloc
 # Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0

 $ ./tools/testing/selftests/drivers/net/xdp.py
 TAP version 13
 1..13
 ok 1 xdp.test_xdp_native_pass_sb
 ok 2 xdp.test_xdp_native_pass_mb
 ok 3 xdp.test_xdp_native_drop_sb
 ok 4 xdp.test_xdp_native_drop_mb
 ok 5 xdp.test_xdp_native_tx_sb
 ok 6 xdp.test_xdp_native_tx_mb
 # Failed run: pkt_sz 2048, offset 1. Last successful run: pkt_sz 1024, offset 256. Reason: Adjustment failed
 ok 7 xdp.test_xdp_native_adjst_tail_grow_data
 ok 8 xdp.test_xdp_native_adjst_tail_shrnk_data
 # Failed run: pkt_sz 512, offset -256. Last successful run: pkt_sz 512, offset -128. Reason: Adjustment failed
 ok 9 xdp.test_xdp_native_adjst_head_grow_data
 # Failed run: pkt_sz (2048) > HDS threshold (1536) and offset 64 > 48
 ok 10 xdp.test_xdp_native_adjst_head_shrnk_data
 ok 11 xdp.test_xdp_native_qstats_pass
 ok 12 xdp.test_xdp_native_qstats_drop
 ok 13 xdp.test_xdp_native_qstats_tx
 # Totals: pass:13 fail:0 xfail:0 xpass:0 skip:0 error:0
====================

Link: https://patch.msgid.link/20251007232653.2099376-1-kuba@kernel.org
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

6 files changed, 87 insertions, 32 deletions

diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
index a1c2db69b198..95fac020eb93 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
@@ -185,13 +185,13 @@ static void fbnic_aggregate_vector_counters(struct fbnic_net *fbn,
 
 	for (i = 0; i < nv->txt_count; i++) {
 		fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub0);
-		fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub1);
+		fbnic_aggregate_ring_xdp_counters(fbn, &nv->qt[i].sub1);
 		fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].cmpl);
 	}
 
 	for (j = 0; j < nv->rxt_count; j++, i++) {
-		fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].sub0);
-		fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].sub1);
+		fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub0);
+		fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub1);
 		fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].cmpl);
 	}
 }
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
index 8f998d26b9a3..2a84bd1d7e26 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
@@ -83,8 +83,16 @@ static void fbnic_mac_init_axi(struct fbnic_dev *fbd)
 
 static void fbnic_mac_init_qm(struct fbnic_dev *fbd)
 {
+	u64 default_meta = FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, ETH_HLEN) |
+			   FBNIC_TWD_FLAG_REQ_COMPLETION;
 	u32 clock_freq;
 
+	/* Configure default TWQ Metadata descriptor */
+	wr32(fbd, FBNIC_QM_TWQ_DEFAULT_META_L,
+	     lower_32_bits(default_meta));
+	wr32(fbd, FBNIC_QM_TWQ_DEFAULT_META_H,
+	     upper_32_bits(default_meta));
+
 	/* Configure TSO behavior */
 	wr32(fbd, FBNIC_QM_TQS_CTL0,
 	     FIELD_PREP(FBNIC_QM_TQS_CTL0_LSO_TS_MASK,
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
index d12b4cad84a5..e95be0e7bd9e 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
@@ -543,17 +543,21 @@ static const struct net_device_ops fbnic_netdev_ops = {
 static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
 				     struct netdev_queue_stats_rx *rx)
 {
+	u64 bytes, packets, alloc_fail, alloc_fail_bdq;
 	struct fbnic_net *fbn = netdev_priv(dev);
 	struct fbnic_ring *rxr = fbn->rx[idx];
 	struct fbnic_dev *fbd = fbn->fbd;
 	struct fbnic_queue_stats *stats;
-	u64 bytes, packets, alloc_fail;
 	u64 csum_complete, csum_none;
+	struct fbnic_q_triad *qt;
 	unsigned int start;
 
 	if (!rxr)
 		return;
 
+	/* fbn->rx points to completion queues */
+	qt = container_of(rxr, struct fbnic_q_triad, cmpl);
+
 	stats = &rxr->stats;
 	do {
 		start = u64_stats_fetch_begin(&stats->syncp);
@@ -564,6 +568,20 @@ static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
 		csum_none = stats->rx.csum_none;
 	} while (u64_stats_fetch_retry(&stats->syncp, start));
 
+	stats = &qt->sub0.stats;
+	do {
+		start = u64_stats_fetch_begin(&stats->syncp);
+		alloc_fail_bdq = stats->bdq.alloc_failed;
+	} while (u64_stats_fetch_retry(&stats->syncp, start));
+	alloc_fail += alloc_fail_bdq;
+
+	stats = &qt->sub1.stats;
+	do {
+		start = u64_stats_fetch_begin(&stats->syncp);
+		alloc_fail_bdq = stats->bdq.alloc_failed;
+	} while (u64_stats_fetch_retry(&stats->syncp, start));
+	alloc_fail += alloc_fail_bdq;
+
 	rx->bytes = bytes;
 	rx->packets = packets;
 	rx->alloc_fail = alloc_fail;
@@ -641,7 +659,8 @@ static void fbnic_get_base_stats(struct net_device *dev,
 
 	rx->bytes = fbn->rx_stats.bytes;
 	rx->packets = fbn->rx_stats.packets;
-	rx->alloc_fail = fbn->rx_stats.rx.alloc_failed;
+	rx->alloc_fail = fbn->rx_stats.rx.alloc_failed +
+		fbn->bdq_stats.bdq.alloc_failed;
 	rx->csum_complete = fbn->rx_stats.rx.csum_complete;
 	rx->csum_none = fbn->rx_stats.rx.csum_none;
 }
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h
index e84e0527c3a9..b0a87c57910f 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h
@@ -68,6 +68,7 @@ struct fbnic_net {
 	/* Storage for stats after ring destruction */
 	struct fbnic_queue_stats tx_stats;
 	struct fbnic_queue_stats rx_stats;
+	struct fbnic_queue_stats bdq_stats;
 	u64 link_down_events;
 
 	/* Time stamping filter config */
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
index cf773cc78e40..b1e8ce89870f 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
@@ -904,7 +904,7 @@ static void fbnic_fill_bdq(struct fbnic_ring *bdq)
 		netmem = page_pool_dev_alloc_netmems(bdq->page_pool);
 		if (!netmem) {
 			u64_stats_update_begin(&bdq->stats.syncp);
-			bdq->stats.rx.alloc_failed++;
+			bdq->stats.bdq.alloc_failed++;
 			u64_stats_update_end(&bdq->stats.syncp);
 
 			break;
@@ -1242,6 +1242,7 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
 	/* Walk the completion queue collecting the heads reported by NIC */
 	while (likely(packets < budget)) {
 		struct sk_buff *skb = ERR_PTR(-EINVAL);
+		u32 pkt_bytes;
 		u64 rcd;
 
 		if ((*raw_rcd & cpu_to_le64(FBNIC_RCD_DONE)) == done)
@@ -1272,37 +1273,38 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
 			/* We currently ignore the action table index */
 			break;
 		case FBNIC_RCD_TYPE_META:
-			if (unlikely(pkt->add_frag_failed))
-				skb = NULL;
-			else if (likely(!fbnic_rcd_metadata_err(rcd)))
+			if (likely(!fbnic_rcd_metadata_err(rcd) &&
+				   !pkt->add_frag_failed)) {
+				pkt_bytes = xdp_get_buff_len(&pkt->buff);
 				skb = fbnic_run_xdp(nv, pkt);
+			}
 
 			/* Populate skb and invalidate XDP */
 			if (!IS_ERR_OR_NULL(skb)) {
 				fbnic_populate_skb_fields(nv, rcd, skb, qt,
 							  &csum_complete,
 							  &csum_none);
-
-				packets++;
-				bytes += skb->len;
-
 				napi_gro_receive(&nv->napi, skb);
 			} else if (skb == ERR_PTR(-FBNIC_XDP_TX)) {
 				pkt_tail = nv->qt[0].sub1.tail;
-				bytes += xdp_get_buff_len(&pkt->buff);
+			} else if (PTR_ERR(skb) == -FBNIC_XDP_CONSUME) {
+				fbnic_put_pkt_buff(qt, pkt, 1);
 			} else {
-				if (!skb) {
+				if (!skb)
 					alloc_failed++;
-					dropped++;
-				} else if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR)) {
+
+				if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR))
 					length_errors++;
-				} else {
+				else
 					dropped++;
-				}
 
 				fbnic_put_pkt_buff(qt, pkt, 1);
+				goto next_dont_count;
 			}
 
+			packets++;
+			bytes += pkt_bytes;
+next_dont_count:
 			pkt->buff.data_hard_start = NULL;
 
 			break;
@@ -1319,8 +1321,6 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv,
 	u64_stats_update_begin(&rcq->stats.syncp);
 	rcq->stats.packets += packets;
 	rcq->stats.bytes += bytes;
-	/* Re-add ethernet header length (removed in fbnic_build_skb) */
-	rcq->stats.bytes += ETH_HLEN * packets;
 	rcq->stats.dropped += dropped;
 	rcq->stats.rx.alloc_failed += alloc_failed;
 	rcq->stats.rx.csum_complete += csum_complete;
@@ -1414,6 +1414,17 @@ void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,
 	BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 4);
 }
 
+void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn,
+				       struct fbnic_ring *bdq)
+{
+	struct fbnic_queue_stats *stats = &bdq->stats;
+
+	/* Capture stats from queues before dissasociating them */
+	fbn->bdq_stats.bdq.alloc_failed += stats->bdq.alloc_failed;
+	/* Remember to add new stats here */
+	BUILD_BUG_ON(sizeof(fbn->rx_stats.bdq) / 8 != 1);
+}
+
 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
 				      struct fbnic_ring *txr)
 {
@@ -1433,8 +1444,8 @@ void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
 	BUILD_BUG_ON(sizeof(fbn->tx_stats.twq) / 8 != 6);
 }
 
-static void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
-					      struct fbnic_ring *xdpr)
+void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
+				       struct fbnic_ring *xdpr)
 {
 	struct fbnic_queue_stats *stats = &xdpr->stats;
 
@@ -1442,9 +1453,7 @@ static void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
 		return;
 
 	/* Capture stats from queues before dissasociating them */
-	fbn->rx_stats.bytes += stats->bytes;
-	fbn->rx_stats.packets += stats->packets;
-	fbn->rx_stats.dropped += stats->dropped;
+	fbn->tx_stats.dropped += stats->dropped;
 	fbn->tx_stats.bytes += stats->bytes;
 	fbn->tx_stats.packets += stats->packets;
 }
@@ -1488,6 +1497,15 @@ static void fbnic_remove_rx_ring(struct fbnic_net *fbn,
 	fbn->rx[rxr->q_idx] = NULL;
 }
 
+static void fbnic_remove_bdq_ring(struct fbnic_net *fbn,
+				  struct fbnic_ring *bdq)
+{
+	if (!(bdq->flags & FBNIC_RING_F_STATS))
+		return;
+
+	fbnic_aggregate_ring_bdq_counters(fbn, bdq);
+}
+
 static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt)
 {
 	page_pool_destroy(qt->sub0.page_pool);
@@ -1507,8 +1525,8 @@ static void fbnic_free_napi_vector(struct fbnic_net *fbn,
 	}
 
 	for (j = 0; j < nv->rxt_count; j++, i++) {
-		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0);
-		fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1);
+		fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub0);
+		fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub1);
 		fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl);
 	}
 
@@ -1707,11 +1725,13 @@ static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn,
 	while (rxt_count) {
 		/* Configure header queue */
 		db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_HPQ_TAIL];
-		fbnic_ring_init(&qt->sub0, db, 0, FBNIC_RING_F_CTX);
+		fbnic_ring_init(&qt->sub0, db, 0,
+				FBNIC_RING_F_CTX | FBNIC_RING_F_STATS);
 
 		/* Configure payload queue */
 		db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_PPQ_TAIL];
-		fbnic_ring_init(&qt->sub1, db, 0, FBNIC_RING_F_CTX);
+		fbnic_ring_init(&qt->sub1, db, 0,
+				FBNIC_RING_F_CTX | FBNIC_RING_F_STATS);
 
 		/* Configure Rx completion queue */
 		db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_RCQ_HEAD];
@@ -2830,8 +2850,8 @@ static int fbnic_queue_start(struct net_device *dev, void *qmem, int idx)
 	real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
 	nv = fbn->napi[idx % fbn->num_napi];
 
-	fbnic_aggregate_ring_rx_counters(fbn, &real->sub0);
-	fbnic_aggregate_ring_rx_counters(fbn, &real->sub1);
+	fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0);
+	fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1);
 	fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl);
 
 	memcpy(real, qmem, sizeof(*real));
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
index 31fac0ba0902..ca37da5a0b17 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
@@ -92,6 +92,9 @@ struct fbnic_queue_stats {
 			u64 csum_none;
 			u64 length_errors;
 		} rx;
+		struct {
+			u64 alloc_failed;
+		} bdq;
 	};
 	u64 dropped;
 	struct u64_stats_sync syncp;
@@ -165,8 +168,12 @@ fbnic_features_check(struct sk_buff *skb, struct net_device *dev,
 
 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,
 				      struct fbnic_ring *rxr);
+void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn,
+				       struct fbnic_ring *rxr);
 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
 				      struct fbnic_ring *txr);
+void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
+				       struct fbnic_ring *xdpr);
 
 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn);
 void fbnic_free_napi_vectors(struct fbnic_net *fbn);